oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: disable mDNS in resolved

Browse Source
This commit is contained in:
Administrator
2025-11-17 17:54:46 +01:00
parent ce121e7d2b
commit 7c4d108026
9 changed files with 173 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
ekman/c0/default.nix
View File

@@ -72,16 +72,17 @@ let
};
# systemd.services.systemd-networkd-wait-online.enable = false;
systemd.network.wait-online.ignoredInterfaces = [ "ibp1s0" ];
systemd.network = {
wait-online.ignoredInterfaces = [ "ibp1s0" ];
# wait-online.enable = false;
networks = {
"40-${host.iface}" = {
matchConfig.Name = host.iface;
address = [ "${host.address}/24" ];
networkConfig = {
DNSDefaultRoute = true;
};
# networkConfig = {
# DNSDefaultRoute = true;
# };
routes = [
{ Gateway = "10.255.241.1"; }
{

1
ekman/default.nix
View File

@@ -49,6 +49,7 @@ let
# domains = [ "ekman.tos.obx" "~." ];
extraConfig = ''
DNSStubListener=no # conflicts with dnsmasq and kubernetes dns
MulticastDNS=no
'';
};

0
gw/hel1-gw-1/default.nix → gw/hel1-1/default.nix
View File

0
gw/hel1-gw-1/hardware-configuration.nix → gw/hel1-1/hardware-configuration.nix
View File

0
gw/hel1-gw-1/ssh_host_key.pub → gw/hel1-1/ssh_host_key.pub
View File

4
gw/hive.nix
View File

@@ -2,6 +2,6 @@ let
pkgs = import <nixpkgs> { };
tos-gw = import ./tos-gw { inherit pkgs; };
vtn-gw = import ./vtn-gw { inherit pkgs; };
hel1-gw-1 = import ./hel1-gw-1 { inherit pkgs; };
hel1-1 = import ./hel1-1 { inherit pkgs; };
in
tos-gw // vtn-gw // hel1-gw-1
tos-gw // vtn-gw // hel1-1

108
gw/vtn-gw/default.nix
View File

@@ -2,65 +2,69 @@
let
name = "vtn-gw";
address = "195.43.37.37";
in {
vtn-gw = { config, pkgs, ... }: {
deployment.tags = [ "gw" ];
deployment.targetHost = address;
in
{
vtn-gw =
{ config, pkgs, ... }:
{
deployment.tags = [ "gw" ];
deployment.targetHost = address;
networking = {
useNetworkd = true;
useDHCP = false;
hostName = name;
firewall = {
allowedTCPPorts = [ 53 ];
allowedUDPPorts = [ 53 ];
extraCommands = ''
'';
networking = {
useNetworkd = true;
useDHCP = false;
hostName = name;
firewall = {
allowedTCPPorts = [ 53 ];
allowedUDPPorts = [ 53 ];
extraCommands = '''';
};
};
};
systemd.network = {
# wait-online.enable = false;
networks."40-enp3s0" = {
matchConfig.Name = "enp3s0";
DHCP = "no";
address = [ "${address}/29" ];
routes = [ { Gateway = "195.43.37.33"; } ];
};
};
systemd.network = {
# wait-online.enable = false;
networks."40-enp3s0" = {
matchConfig.Name = "enp3s0";
DHCP = "no";
address = [ "${address}/29" ];
routes = [ { Gateway = "195.43.37.33"; } ];
};
};
programs.ssh.knownHosts = {
vtn-gw = {
hostNames = [
name "${name}.vtn.obx" address
programs.ssh.knownHosts = {
vtn-gw = {
hostNames = [
name
"${name}.vtn.obx"
address
];
publicKeyFile = ./ssh_host_key.pub;
};
};
features = {
os = {
externalInterface = "enp3s0";
};
};
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "both";
extraUpFlags = [
"--login-server=https://headscale-router.adm.oceanbox.io"
"--accept-dns=true"
"--accept-routes=true"
"--snat-subnet-routes=false"
"--advertise-routes=172.16.239.0/24,172.16.238.0/24"
"--advertise-tags=tag:vtn-router"
];
publicKeyFile = ./ssh_host_key.pub;
};
};
features = {
os = {
externalInterface = "enp3s0";
};
};
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "both";
extraUpFlags = [
"--login-server=https://headscale-router.adm.oceanbox.io"
"--accept-dns=true"
"--accept-routes=true"
"--snat-subnet-routes=false"
"--advertise-routes=172.16.239.0/24,172.16.239.0/24"
"--advertise-tags=tag:vtn-router"
imports = [
./hardware-configuration.nix
../default.nix
];
};
imports = [
./hardware-configuration.nix
../default.nix
];
};
}

2
rossby/c0/default.nix
View File

@@ -65,8 +65,8 @@ let
# systemd.services.systemd-networkd-wait-online.enable = false;
systemd.network.wait-online.ignoredInterfaces = [ "ibp1s0" ];
systemd.network = {
wait-online.ignoredInterfaces = [ "ibp1s0" ];
# wait-online.enable = false;
networks = {
"40-${host.iface}" = {

187
rossby/default.nix
View File

@@ -1,4 +1,9 @@
{ pkgs, lib, config, ... }:
{
pkgs,
lib,
config,
...
}:
with lib;
let
cfg = config.features.host;
@@ -23,7 +28,11 @@ let
loader.systemd-boot.enable = true;
loader.efi.canTouchEfiVariables = true;
# kernelPackages = pkgs.linuxKernel.packages.linux_6_9;
kernelModules = [ "ib_umad" "ib_ipoib" "ceph" ];
kernelModules = [
"ib_umad"
"ib_ipoib"
"ceph"
];
# kernelParams = [
# "console=ttyS0,115200"
# "console=tty0"
@@ -40,6 +49,7 @@ let
# domains = [ "rossby.vtn.obx" "~." ];
extraConfig = ''
DNSStubListener=no # conflicts with dnsmasq and kubernetes dns
MulticastDNS=no
'';
};
@@ -51,14 +61,14 @@ let
i18n = {
defaultLocale = "en_US.UTF-8";
extraLocaleSettings = {
LC_CTYPE="en_DK.UTF-8";
LC_TIME="en_DK.UTF-8";
LC_PAPER="en_DK.UTF-8";
LC_NAME="en_DK.UTF-8";
LC_ADDRESS="en_DK.UTF-8";
LC_TELEPHONE="en_DK.UTF-8";
LC_MEASUREMENT="en_DK.UTF-8";
LC_IDENTIFICATION="en_DK.UTF-8";
LC_CTYPE = "en_DK.UTF-8";
LC_TIME = "en_DK.UTF-8";
LC_PAPER = "en_DK.UTF-8";
LC_NAME = "en_DK.UTF-8";
LC_ADDRESS = "en_DK.UTF-8";
LC_TELEPHONE = "en_DK.UTF-8";
LC_MEASUREMENT = "en_DK.UTF-8";
LC_IDENTIFICATION = "en_DK.UTF-8";
};
};
@@ -94,8 +104,11 @@ let
networking = rec {
useDHCP = false;
domain = "rossby.vtn.obx";
nameservers = [ "195.43.37.37" "172.16.239.221" ];
search = [];
nameservers = [
"195.43.37.37"
"172.16.239.221"
];
search = [ ];
extraHosts = import ../hosts.nix;
firewall.extraCommands = ''
iptables -I INPUT -s 172.16.239.0/24 -j ACCEPT
@@ -104,7 +117,7 @@ let
'';
};
environment.variables = {};
environment.variables = { };
# systemd.services."serial-getty@ttyS0".enable = true;
# environment.etc."beegfs/connauthfile" = {
@@ -125,45 +138,45 @@ let
};
system.activationScripts = {
kraken-permissions.text = ''
chmod 755 /work/kraken
'';
kraken-permissions.text = ''
chmod 755 /work/kraken
'';
};
};
slurm = {
features.hpc.slurm = {
enable = true;
client = true;
clusterName = "rossby";
controlMachine = "rossby-manage(172.16.239.221)";
dbdHost = "slurm-accounting";
# dbdExtraConfig = ''
# DbdAddr=rossby-manage.rossby.vtn.obx
# '';
mungeKey = ../munge.key;
jwtKey = ../jwt_hs256.key;
# slurmKey = ../slurm.key;
# pkey = "0x7666";
mailDomain = "hpc.oceanbox.io";
nodeName = [
"c0-[1-20] Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=382000 TmpDisk=400000 State=UNKNOWN"
"rossby Sockets=1 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=40000 TmpDisk=400000 State=UNKNOWN"
"rossby-manage Sockets=1 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=60000 TmpDisk=400000 State=UNKNOWN"
];
partitionName = [
"batch Nodes=c0-[1-20] Default=YES MaxTime=INFINITE State=UP"
"login Nodes=rossby MaxTime=1:00:00 State=UP"
"short Nodes=c0-[1-10] MaxTime=INFINITE State=UP"
"long Nodes=c0-[11-12] MaxTime=INFINITE State=UP"
# "drifters Nodes=c0-[19-2] MaxTime=INFINITE State=UP OverSubscribe=FORCE:4"
];
enable = true;
client = true;
clusterName = "rossby";
controlMachine = "rossby-manage(172.16.239.221)";
dbdHost = "slurm-accounting";
# dbdExtraConfig = ''
# DbdAddr=rossby-manage.rossby.vtn.obx
# '';
mungeKey = ../munge.key;
jwtKey = ../jwt_hs256.key;
# slurmKey = ../slurm.key;
# pkey = "0x7666";
mailDomain = "hpc.oceanbox.io";
nodeName = [
"c0-[1-20] Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=382000 TmpDisk=400000 State=UNKNOWN"
"rossby Sockets=1 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=40000 TmpDisk=400000 State=UNKNOWN"
"rossby-manage Sockets=1 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=60000 TmpDisk=400000 State=UNKNOWN"
];
partitionName = [
"batch Nodes=c0-[1-20] Default=YES MaxTime=INFINITE State=UP"
"login Nodes=rossby MaxTime=1:00:00 State=UP"
"short Nodes=c0-[1-10] MaxTime=INFINITE State=UP"
"long Nodes=c0-[11-12] MaxTime=INFINITE State=UP"
# "drifters Nodes=c0-[19-2] MaxTime=INFINITE State=UP OverSubscribe=FORCE:4"
];
};
};
compute = {
system.activationScripts = {
mkWorkDir.text = "mkdir -p /work";
mkWorkDir.text = "mkdir -p /work";
};
cluster.slurm = true;
features = {
@@ -202,7 +215,7 @@ let
};
system.activationScripts = {
copyCaKey.text = "cp ${./ca}/ca-key.pem /var/lib/kubernetes/secrets";
copyCaKey.text = "cp ${./ca}/ca-key.pem /var/lib/kubernetes/secrets";
};
services.kubernetes.kubelet.extraSANs = mkSANs {
@@ -218,67 +231,83 @@ let
gid = 0;
text = ''
172.16.239.210
'' + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes;
''
+ builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes;
};
programs.ssh.knownHosts = {
rossby-manage = {
hostNames = [
"rossby-manage" "rossby-manage.rossby.vtn.obx" "172.16.239.221" "10.16.239.221"
"rossby-manage"
"rossby-manage.rossby.vtn.obx"
"172.16.239.221"
"10.16.239.221"
];
publicKeyFile = ./manage/ssh_host_key.pub;
};
rossby-login = {
hostNames = [
"rossby" "rossby-login" "rossby.rossby.vtn.obx" "172.16.239.222" "10.16.239.222"
"rossby"
"rossby-login"
"rossby.rossby.vtn.obx"
"172.16.239.222"
"10.16.239.222"
];
publicKeyFile = ./login/ssh_host_key.pub;
};
fs-work = {
hostNames = [
"fs-work" "fs-work.rossby.vtn.obx" "172.16.239.210"
"fs-work"
"fs-work.rossby.vtn.obx"
"172.16.239.210"
];
publicKeyFile = ./fs-work/ssh_host_key.pub;
};
} // builtins.foldl' (a: x:
let n = toString x.idx;
in a // {
"${x.name}" = {
hostNames = [
"${x.name}"
"${x.name}.rossby.vtn.obx"
"172.16.239.${n}"
"10.16.239.${n}"
];
publicKeyFile = x.pubkey;
};
}) {} computeNodes;
}
// builtins.foldl' (
a: x:
let
n = toString x.idx;
in
a
// {
"${x.name}" = {
hostNames = [
"${x.name}"
"${x.name}.rossby.vtn.obx"
"172.16.239.${n}"
"10.16.239.${n}"
];
publicKeyFile = x.pubkey;
};
}
) { } computeNodes;
environment.systemPackages = [
openssh-shosts
pkgs.inotify-tools
pkgs.ceph
pkgs.ceph-client
openssh-shosts
pkgs.inotify-tools
pkgs.ceph
pkgs.ceph-client
];
security.wrappers = {
ssh-keysign = {
source = "${openssh-shosts}/libexec/ssh-keysign";
owner = "root";
group = "root";
permissions = "u+rs,g+rx,o+rx";
};
ssh-keysign = {
source = "${openssh-shosts}/libexec/ssh-keysign";
owner = "root";
group = "root";
permissions = "u+rs,g+rx,o+rx";
};
};
# Use nvd to get package diff before apply
system.activationScripts.system-diff = {
supportsDryActivation = true; # safe: only outputs to stdout
text = ''
export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
if [ -e /run/current-system ]; then
${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
fi
'';
supportsDryActivation = true; # safe: only outputs to stdout
text = ''
export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
if [ -e /run/current-system ]; then
${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
fi
'';
};
};
@@ -288,7 +317,8 @@ let
doCheck = false; # the tests take hours
});
in {
in
{
options.cluster = {
compute = mkEnableOption "Enable compute node configs";
};
@@ -315,4 +345,3 @@ in {
../users.nix
];
}