oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add ekman

Browse Source
This commit is contained in:
Jonas Juselius
2022-08-18 15:14:32 +02:00
parent f072bbcded
commit 88b56b609e
6 changed files with 121 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
clusters/ekman/cluster.nix
View File

@@ -12,17 +12,24 @@ let
configuration = {
system.autoUpgrade.enable = lib.mkForce false;
nixpkgs.overlays = [
(import ./overlays.nix)
];
boot = {
loader.systemd-boot.enable = true;
loader.efi.canTouchEfiVariables = true;
kernelPackages = pkgs.linuxPackages_5_4;
kernelModules = [ "ib_umad" "ib_ipoib" ];
kernelParams = [
"console=ttyS0,115200"
"console=tty0"
];
# kernelParams = [
# "console=ttyS0,115200"
# "console=tty0"
# ];
};
services.udev.extraRules = ''
KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x3666"
'';
console = {
font = "Lat2-Terminus16";
@@ -45,6 +52,34 @@ let
time.timeZone = "Europe/Oslo";
programs.msmtp = {
enable = true;
accounts = {
default = {
auth = false;
tls = false;
tls_starttls = false;
port = 24;
from = "ekman@oceanbox.io";
host = "smtpgw.itpartner.no";
# user = "utvikling";
# password = "S0m3rp0m@de#21!";
};
};
defaults = {
aliases = "/etc/aliases";
};
};
environment.etc = {
"aliases" = {
text = ''
root: jonas.juselius@oceanbox.io
'';
mode = "0644";
};
};
features = {
os = {
# boot.uefi = true;
@@ -54,14 +89,6 @@ let
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
];
docker.enable = false;
mailRelay = {
enable = true;
adminEmail = "jonas.juselius@tromso.serit.no";
mailDomain = "itpartner.no";
mailGateway = "smtpgw.itpartner.no:465";
mailAuthUser = "utvikling";
};
};
cachix.enable = false;
@@ -89,9 +116,9 @@ let
beegfs = {
bee0-0 = {
mgmtdHost = "bee0-0";
connAuthFile = "";
connAuthFile = "/etc/beegfs/connauthfile";
client = {
enable = true;
enable = false;
mountPoint = "/work";
};
};
@@ -107,11 +134,11 @@ let
cidr = "10.100.0.0/16";
master = {
name = "ekman";
address = "10.255.240.200";
extraSANs = [ "ekman.local" ];
address = "10.255.241.8";
extraSANs = [ "ekman.local" "ekman.oceanbox.io" ];
};
ingressNodes = [
"ekman.local"
"ekman.oceanbox.io"
];
fileserver = "bee0-0";
charts = {
@@ -128,13 +155,13 @@ let
};
networking = {
domain = mkDefault "oceanbox.io";
defaultGateway = mkDefault "10.1.61.1";
domain = mkDefault "cluster.local";
defaultGateway = mkDefault "10.255.241.1";
nameservers = mkDefault [ "8.8.8.8" ];
search = mkDefault [ "local" ];
extraHosts = import ./hosts.nix;
firewall.extraCommands = ''
iptables -I INPUT -s 10.255.240.0/24 -j ACCEPT
iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
'';
};
@@ -142,12 +169,19 @@ let
systemd.services."serial-getty@ttyS0".enable = true;
environment.etc."beegfs/connauthfile" = {
source = ./connauthfile;
mode = "0400";
uid = 0;
gid = 0;
};
nix = {
maxJobs = 32;
trustedUsers = [ "@wheel" ];
# binaryCachePublicKeys = [
# "ekman-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig=="
# ];
binaryCachePublicKeys = [
"ekman:pka41J3q4j9ZC3dr4y+sDN9uMW0pAxoWeCkrzUlqcZs="
];
};
};
@@ -155,40 +189,25 @@ let
deployment.targetHost = cfg.address;
};
# i40efix = {
# boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
# kernel = pkgs.linuxPackages_5_4.kernel;
# }; in {
# extraModulePackages = [ kernelExtras.i40e2 ];
# kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
# };
# };
i40efix = {
boot = {
extraModulePackages = [];
kernelModules = [ "ib_umad" "ib_ipoib" ];
};
};
shosts = {
environment.etc."ssh/shosts.equiv" = {
mode = "0644";
uid = 0;
gid = 0;
text = ''
10.255.240.200
10.255.240.201
10.255.241.8
10.255.241.11
'';
};
programs.ssh.knownHosts = {
ekman = {
hostNames = [
"ekman" "ekman.local" "ekman.oceanbox.io" "10.255.240.200"
"ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.8"
];
publicKeyFile = ./pubkeys/ekman.pub;
};
c0-1 = { hostNames = [ "c0-1" "c0-1.local" "10.255.240.201" "10.255.241.201" ]; publicKeyFile = ./pubkeys/c0-1.pub; };
c0-1 = { hostNames = [ "c0-1" "c0-1.cluster.local" "10.255.241.11" "10.255.243.11" ]; publicKeyFile = ./pubkeys/c0-1.pub; };
};
environment.systemPackages = [ openssh-shosts ];
@@ -205,6 +224,7 @@ let
openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {
buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ];
doCheck = false; # the tests take hours
});
myvnc =
@@ -243,8 +263,6 @@ let
in {
options.node = {
i40efix = mkEnableOption "Apply fix for i40e driver";
myvnc = mkEnableOption "Enable myvnc script";
};
@@ -255,8 +273,6 @@ in {
shosts
(mkIf config.node.i40efix i40efix)
(mkIf config.node.myvnc myvnc)
];

1
clusters/ekman/connauthfile Normal file
View File

@@ -0,0 +1 @@
莵q丘

82
clusters/ekman/default.nix
View File

@@ -7,20 +7,21 @@ let
pkgs = import <nixpkgs> {};
etcdNodes = {
c0-0 = "10.255.240.200";
c0-1 = "10.255.240.201";
ekman = "10.255.241.8";
nsf0-0 = "10.255.241.9";
bee0-0 = "10.255.241.10";
};
etcdCluster = {
enable = true;
existing = false;
existing = true;
nodes = etcdNodes;
};
nodes =
with builtins;
let nodes = genList (n: n + 1) 1; in
map (n: ({ name = "c0-${toString n}"; address = "10.255.240.20${toString n}"; })) nodes;
map (n: ({ name = "c0-${toString n}"; address = "10.255.241.${toString (n + 10)}"; })) nodes;
ekman = {
# deployment.tags = [ "frontend" ];
@@ -35,16 +36,16 @@ let
features = {
host = {
address = "10.255.240.200";
name = "c0-0";
address = "10.255.241.8";
name = "ekman";
};
os = {
externalInterface = "eno1";
externalInterface = "enp33s0f0np0";
nfs.enable = true;
nfs.exports = ''
/exports 10.255.240.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
@@ -106,28 +107,28 @@ let
interfaces.enp33s0f0np0 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.240.200";
address = "10.255.241.8";
prefixLength = 24;
} ];
};
# interfaces.enp33s0f0np1 = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = "10.1.61.100";
# prefixLength = 24;
# } ];
# };
# interfaces.ibp59s0 = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = "10.255.241.200";
# prefixLength = 24;
# } ];
interfaces.enp33s0f0np1 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.242.2";
prefixLength = 24;
} ];
};
defaultGateway = "10.255.240.1";
interfaces."ibp1s0.3666" = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.243.8";
prefixLength = 24;
} ];
};
defaultGateway = "10.255.241.1";
firewall.extraCommands = ''
iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.255.241.0/24 -j MASQUERADE
iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';
};
@@ -161,7 +162,7 @@ let
};
nix.extraOptions = ''
secret-key-files = /etc/nix/ekman.private
secret-key-files = /etc/nix/ekman.key
'';
services.xserver = {
@@ -255,7 +256,7 @@ let
fileSystems = {
"/frontend" = {
device = "10.255.240.200:/home";
device = "10.255.241.8:/home";
fsType = "nfs";
options = [
"soft"
@@ -286,7 +287,7 @@ let
mkCompute = host:
let
ipoib = builtins.replaceStrings [".240."] [".241."] host.address;
ipoib = builtins.replaceStrings [".241."] [".243."] host.address;
hw = ./hw + "/${host.name}.nix";
in {
"${host.name}" = {
@@ -298,7 +299,6 @@ let
};
node = {
i40efix = true;
};
networking = {
@@ -309,20 +309,20 @@ let
address = host.address;
prefixLength = 24;
} ];
# ipv4.routes = [ {
# address = "10.1.62.2";
# prefixLength = 32;
# via = "10.1.61.100";
# } ];
ipv4.routes = [ {
address = "10.255.242.2";
prefixLength = 32;
via = "10.1.241.8";
} ];
};
# interfaces.ibp65s0 = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = ipoib;
# prefixLength = 24;
# } ];
# };
interfaces."ibp1s0.3666" = {
useDHCP = false;
ipv4.addresses = [ {
address = ipoib;
prefixLength = 24;
} ];
};
};
imports = [ ./cluster.nix hw ];
}

11
clusters/ekman/hosts.nix
View File

@@ -1,10 +1,11 @@
''
10.255.240.200 frontend frontend.local c0-0 ekman ekman.oceanbox.io
10.255.240.201 c0-1 c0-1.local
10.255.240.200 ekman ekman.cluster.local
10.255.240.200 etcd0 etcd0.cluster.local
10.255.240.201 c0-1 c0-1.cluster.local
# 10.1.61.80 bee0-0 bee0-0.local
# 10.1.61.80 bee0-0 bee0-0.cluster.local
# 10.1.63.101 ib0-1 ib0-1.local
# 10.1.63.101 ib0-1 ib0-1.cluster.local
# 10.1.63.80 ibmds0-0 ibmds0-0.local
# 10.1.63.80 ibmds0-0 ibmds0-0.cluster.local
''

11
clusters/ekman/overlays.nix Normal file
View File

@@ -0,0 +1,11 @@
self: super:
let
msmtp = super.msmtp.overrideAttrs (attrs: rec {
configureFlags = attrs.configureFlags ++ [ "--with-tls=openssl" ];
buildInputs = attrs.buildInputs ++ [ super.openssl ];
});
in
{
# inherit msmtp;
}

30
clusters/ekman/users.nix
View File

@@ -1,7 +1,6 @@
{ pkgs, ... }:
{
users.groups = {
admin = { gid = 10000; };
jonas = { gid = 1000; };
olean = { gid = 1001; };
frankgaa = { gid = 1002; };
@@ -21,35 +20,6 @@
};
users.users = {
admin = {
description = "Administrator";
home = "/home/admin";
group = "admin";
extraGroups = [
"users"
"wheel"
"root"
"adm"
"admin"
"cdrom"
"fuse"
"wireshark"
"libvirtd"
"networkmanager"
"tty"
"keys"
];
uid = 10000;
isNormalUser = true;
createHome = false;
useDefaultShell = false;
shell = pkgs.fish;
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas-3"
];
};
jonas = {
description = "Jonas Juselius";
home = "/home/jonas";