From 670c75953dd3781b3260841bbc34175acab87666 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Thu, 9 Oct 2025 20:46:41 +0200 Subject: [PATCH] feat: convert rossby to networkd and resolved --- install-anywhere/flake.nix | 2 +- rossby/c0/default.nix | 45 +++++++++++++++----------------------- rossby/c0/nodes.nix | 4 ++-- rossby/default.nix | 4 ++-- rossby/fs-work/default.nix | 25 ++++++++++++--------- rossby/hosts.nix | 29 ++++++++++++++++++++++++ rossby/login/default.nix | 12 +++++----- rossby/manage/default.nix | 22 +++++++++---------- 8 files changed, 84 insertions(+), 59 deletions(-) diff --git a/install-anywhere/flake.nix b/install-anywhere/flake.nix index 942bd65..f45bc4b 100644 --- a/install-anywhere/flake.nix +++ b/install-anywhere/flake.nix @@ -1,5 +1,5 @@ { - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-25.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; inputs.disko.url = "github:nix-community/disko"; inputs.disko.inputs.nixpkgs.follows = "nixpkgs"; diff --git a/rossby/c0/default.nix b/rossby/c0/default.nix index b38aece..606fc73 100644 --- a/rossby/c0/default.nix +++ b/rossby/c0/default.nix @@ -37,6 +37,7 @@ let name = host.name; address = host.address; }; + os.networkmanager.enable = false; os.externalInterface = host.iface; hpc.computeNode = true; # hpc.knem = true; @@ -49,36 +50,26 @@ let # ''; networking = { + useNetworkd = true; hostName = host.name; useDHCP = false; - # interfaces.enp65s0f0 = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = host.address; - # prefixLength = 24; - # } ]; - # ipv4.routes = [ { - # address = "10.255.242.0"; - # prefixLength = 24; - # via = "10.255.241.100"; - # } ]; - # }; - interfaces."${host.iface}" = { - useDHCP = false; - ipv4.addresses = [ { - address = host.address; - prefixLength = 24; - } ]; - }; - # interfaces."ibp1s0.7666" = { - # interfaces."ibp1s0" = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = host.ipoib; - # prefixLength = 24; - # } ]; - # }; }; + + systemd.network = { + networks = { + "40-${host.iface}" = { + matchConfig.Name = host.iface; + address = [ "${host.address}/24" ]; + routes = [ { Gateway = "172.16.239.1"; } ]; + }; + # "ibp1s0.7666" = { + "45-ibp1s0" = { + matchConfig.Name = "ibp1s0"; + address = [ "${host.ipoib}/24" ]; + }; + }; + }; + imports = [ hw ../default.nix diff --git a/rossby/c0/nodes.nix b/rossby/c0/nodes.nix index fb745d0..662a115 100644 --- a/rossby/c0/nodes.nix +++ b/rossby/c0/nodes.nix @@ -4,9 +4,9 @@ let map (n: ( rec { idx = 110 + n; - iface = if n > 1 && n < 11 then "enp34s0np0" else "enp33s0np0"; + iface = if n > 4 && n < 11 then "enp34s0np0" else "enp33s0np0"; name = "c0-${toString n}"; address = "172.16.239.${toString idx}"; - ipoib = "10.1.6.${toString idx}"; + ipoib = "10.16.239.${toString idx}"; pubkey = ./. + "/ssh_host_key.d/c0-${toString n}.pub"; })) nodes diff --git a/rossby/default.nix b/rossby/default.nix index 458546e..194a227 100644 --- a/rossby/default.nix +++ b/rossby/default.nix @@ -33,6 +33,7 @@ let services.resolved = { enable = true; dnssec = "false"; + domains = [ "cluster.local" ]; fallbackDns = [ "1.1.1.1" "1.0.0.1" @@ -109,8 +110,7 @@ let networking = { useDHCP = false; domain = mkDefault "cluster.local"; - defaultGateway = mkDefault "172.16.239.1"; - nameservers = mkDefault [ "8.8.8.8" ]; + nameservers = [ "8.8.8.8" "8.8.4.4" ]; search = mkDefault []; extraHosts = import ./hosts.nix; firewall.extraCommands = '' diff --git a/rossby/fs-work/default.nix b/rossby/fs-work/default.nix index 3f6bfe4..39858d2 100644 --- a/rossby/fs-work/default.nix +++ b/rossby/fs-work/default.nix @@ -53,6 +53,7 @@ in { }; os = { + networkmanager.enable = false; externalInterface = "enp1s0np0"; nfs.enable = true; nfs.exports = '' @@ -97,18 +98,13 @@ in { }; networking = { + useNetworkd = true; + useDHCP = false; hostName = name; - interfaces.enp129s0f0.useDHCP = false; - interfaces.enp1s0np0 = { - useDHCP = false; - ipv4.addresses = [ - { - address = address; - prefixLength = 24; - } - ]; + defaultGateway = { + interface = "enp1s0np0"; + address = "172.16.239.1"; }; - defaultGateway = "172.16.239.1"; firewall = { allowedTCPPorts = []; allowedUDPPorts = []; @@ -118,6 +114,15 @@ in { }; }; + systemd.network = { + networks."40-enp1s0np0" = { + matchConfig.Name = "enp1s0np0"; + DHCP = "no"; + address = [ "${address}/24" ]; + routes = [ { Gateway = "172.16.239.1"; } ]; + }; + }; + services.rpcbind.enable = true; fileSystems = { diff --git a/rossby/hosts.nix b/rossby/hosts.nix index 2008d53..43f0e67 100644 --- a/rossby/hosts.nix +++ b/rossby/hosts.nix @@ -1,2 +1,31 @@ '' +# 172.16.241.2 rossby-gw rossby-gw.cluster.local rossby-gw.cluster.local +# 172.16.241.3 front-gw front-gw.cluster.local front-gw.cluster.local + +172.16.239.222 rossby rossby-login rossby-login.cluster.local rossby.cluster.local +172.16.239.210 fs-work fs-work.cluster.local +172.16.239.221 etcd0 etcd0.cluster.local +172.16.239.222 etcd1 etcd1.cluster.local +172.16.239.210 etcd2 etcd2.cluster.local + +172.16.239.111 c0-1 c0-1.cluster.local +172.16.239.112 c0-2 c0-2.cluster.local +172.16.239.113 c0-3 c0-3.cluster.local +172.16.239.114 c0-4 c0-4.cluster.local +172.16.239.115 c0-5 c0-5.cluster.local +172.16.239.116 c0-6 c0-6.cluster.local +172.16.239.117 c0-7 c0-7.cluster.local +172.16.239.118 c0-8 c0-8.cluster.local +172.16.239.119 c0-9 c0-9.cluster.local +172.16.239.120 c0-10 c0-10.cluster.local +172.16.239.121 c0-11 c0-11.cluster.local +172.16.239.122 c0-12 c0-12.cluster.local +172.16.239.123 c0-13 c0-13.cluster.local +172.16.239.124 c0-14 c0-14.cluster.local +172.16.239.125 c0-15 c0-15.cluster.local +172.16.239.126 c0-16 c0-16.cluster.local +172.16.239.127 c0-17 c0-17.cluster.local +172.16.239.128 c0-18 c0-18.cluster.local +172.16.239.129 c0-19 c0-18.cluster.local +172.16.239.130 c0-20 c0-18.cluster.local '' diff --git a/rossby/login/default.nix b/rossby/login/default.nix index 0b8a888..df37d1f 100644 --- a/rossby/login/default.nix +++ b/rossby/login/default.nix @@ -40,6 +40,7 @@ in myvnc.enable = false; os = { + networkmanager.enable = false; externalInterface = "enp65s0np0"; nfs.enable = true; nfs.exports = '' @@ -91,13 +92,13 @@ in # "172.19.255.0/24" # ]; infiniband-exporter = { - enable = true; + enable = false; nameMap = '' # 0xe8ebd3030024981e "c0-1" ''; }; slurm-exporter = { - enable = true; + enable = false; port = 6080; }; }; @@ -111,8 +112,9 @@ in services.flannel.iface = "enp65s0np0"; networking = { - useDHCP = false; + useNetworkd = true; hostName = name; + nameservers = [ "8.8.8.8" "8.8.4.4" ]; firewall = { allowedTCPPorts = [ 6443 ]; extraCommands = '' @@ -122,10 +124,8 @@ in }; systemd.network = { - wait-online.enable = false; - networks.wired = { + networks."40-enp65s0np0" = { matchConfig.Name = "enp65s0np0"; - DHCP = "no"; address = [ "${address}/24" ]; routes = [ { Gateway = "172.16.239.1"; } ]; }; diff --git a/rossby/manage/default.nix b/rossby/manage/default.nix index cc49c38..5906b13 100644 --- a/rossby/manage/default.nix +++ b/rossby/manage/default.nix @@ -145,12 +145,7 @@ in { networking = { useNetworkd = true; - useDHCP = false; hostName = name; - defaultGateway = { - interface = "enp65s0np0"; - address = "172.16.239.1"; - }; firewall = { allowedTCPPorts = [ 6443 4725 ]; extraCommands = '' @@ -167,14 +162,14 @@ in { }; systemd.network = { - wait-online.enable = false; networks = { - enp65s0np0 = { - DHCP = "no"; + "40-enp65s0np0" = { matchConfig.Name = "enp65s0np0"; + DHCP = "no"; address = [ "${address}/24" ]; + routes = [ { Gateway = "172.16.239.1"; } ]; }; - ipoib = { + "45-ibp1s0" = { matchConfig.Name = "ibp1s0"; DHCP = "no"; address = [ "${ipoib}/24" ]; @@ -183,6 +178,7 @@ in { }; services.resolved = { + # DNS=[::1]:53 extraConfig = '' DNSStubListener=no ''; @@ -239,10 +235,14 @@ in { ''; }; - services.dnsmasq.enable = true; + services.coredns.enable = false; + services.coredns.config = { + + }; + services.dnsmasq.enable = false; services.dnsmasq.settings = { - domain = [ "cluster.local" ]; address = [ + "/rossby-manage.cluster.local/172.16.239.221" "/slurmctld.cluster.local/127.0.0.1" ]; srv-host = "_slurmctld._tcp.cluster.local,slurmctld.cluster.local,6817,0,5";