feat: convert rossby to networkd and resolved

install-anywhere/flake.nix

@@ -1,5 +1,5 @@
 {
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-25.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
   inputs.disko.url = "github:nix-community/disko";
   inputs.disko.inputs.nixpkgs.follows = "nixpkgs";
 

rossby/c0/default.nix

@@ -37,6 +37,7 @@ let
           name = host.name;
           address = host.address;
         };
+        os.networkmanager.enable = false;
         os.externalInterface = host.iface;
         hpc.computeNode = true;
         # hpc.knem = true;
@@ -49,36 +50,26 @@ let
       # '';
 
       networking = {
+        useNetworkd = true;
         hostName = host.name;
         useDHCP = false;
-        # interfaces.enp65s0f0 = {
-        #   useDHCP = false;
-        #   ipv4.addresses = [ {
-        #     address = host.address;
-        #     prefixLength = 24;
-        #   } ];
-          # ipv4.routes = [ {
-          #   address = "10.255.242.0";
-          #   prefixLength = 24;
-          #   via = "10.255.241.100";
-          # } ];
-        # };
-        interfaces."${host.iface}" = {
-          useDHCP = false;
-          ipv4.addresses = [ {
-            address = host.address;
-            prefixLength = 24;
-          } ];
-        };
-        # interfaces."ibp1s0.7666" = {
-        # interfaces."ibp1s0" = {
-        #   useDHCP = false;
-        #   ipv4.addresses = [ {
-        #     address = host.ipoib;
-        #     prefixLength = 24;
-        #   } ];
-        # };
       };
+
+      systemd.network = {
+        networks = {
+          "40-${host.iface}" = {
+            matchConfig.Name = host.iface;
+            address = [ "${host.address}/24" ];
+            routes = [ { Gateway = "172.16.239.1"; } ];
+          };
+        # "ibp1s0.7666" = {
+          "45-ibp1s0" = {
+            matchConfig.Name = "ibp1s0";
+            address = [ "${host.ipoib}/24" ];
+          };
+        };
+      };
+
       imports = [
           hw
           ../default.nix

rossby/c0/nodes.nix

@@ -4,9 +4,9 @@ let
 map (n: (
   rec {
     idx = 110 + n;
-    iface = if n > 1 && n < 11 then "enp34s0np0" else "enp33s0np0";
+    iface = if n > 4 && n < 11 then "enp34s0np0" else "enp33s0np0";
     name = "c0-${toString n}";
     address = "172.16.239.${toString idx}";
-    ipoib = "10.1.6.${toString idx}";
+    ipoib = "10.16.239.${toString idx}";
     pubkey = ./. +  "/ssh_host_key.d/c0-${toString n}.pub";
   })) nodes

rossby/default.nix

@@ -33,6 +33,7 @@ let
     services.resolved = {
       enable = true;
       dnssec = "false";
+      domains = [ "cluster.local" ];
       fallbackDns = [
         "1.1.1.1"
         "1.0.0.1"
@@ -109,8 +110,7 @@ let
     networking = {
       useDHCP = false;
       domain = mkDefault "cluster.local";
-      defaultGateway = mkDefault "172.16.239.1";
+      nameservers = [ "8.8.8.8" "8.8.4.4" ];
-      nameservers = mkDefault [ "8.8.8.8" ];
       search = mkDefault [];
       extraHosts = import ./hosts.nix;
       firewall.extraCommands = ''

rossby/fs-work/default.nix

@@ -53,6 +53,7 @@ in {
       };
 
       os = {
+        networkmanager.enable = false;
         externalInterface = "enp1s0np0";
         nfs.enable = true;
         nfs.exports = ''
@@ -97,18 +98,13 @@ in {
     };
 
     networking = {
+      useNetworkd = true;
+      useDHCP = false;
       hostName = name;
-      interfaces.enp129s0f0.useDHCP = false;
+      defaultGateway = {
-      interfaces.enp1s0np0 = {
+        interface = "enp1s0np0";
-        useDHCP = false;
+        address = "172.16.239.1";
-        ipv4.addresses = [
-          {
-            address = address;
-            prefixLength = 24;
-          }
-        ];
       };
-      defaultGateway = "172.16.239.1";
       firewall = {
         allowedTCPPorts = [];
         allowedUDPPorts = [];
@@ -118,6 +114,15 @@ in {
       };
     };
 
+    systemd.network = {
+      networks."40-enp1s0np0" = {
+        matchConfig.Name = "enp1s0np0";
+        DHCP = "no";
+        address = [ "${address}/24" ];
+        routes = [ { Gateway = "172.16.239.1"; } ];
+      };
+    };
+
     services.rpcbind.enable = true;
 
     fileSystems = {

rossby/hosts.nix

@@ -1,2 +1,31 @@
 ''
+#    172.16.241.2 rossby-gw rossby-gw.cluster.local rossby-gw.cluster.local
+#    172.16.241.3 front-gw front-gw.cluster.local front-gw.cluster.local
+
+172.16.239.222 rossby rossby-login rossby-login.cluster.local rossby.cluster.local
+172.16.239.210 fs-work fs-work.cluster.local
+172.16.239.221 etcd0 etcd0.cluster.local
+172.16.239.222 etcd1 etcd1.cluster.local
+172.16.239.210 etcd2 etcd2.cluster.local
+
+172.16.239.111 c0-1  c0-1.cluster.local
+172.16.239.112 c0-2  c0-2.cluster.local
+172.16.239.113 c0-3  c0-3.cluster.local
+172.16.239.114 c0-4  c0-4.cluster.local
+172.16.239.115 c0-5  c0-5.cluster.local
+172.16.239.116 c0-6  c0-6.cluster.local
+172.16.239.117 c0-7  c0-7.cluster.local
+172.16.239.118 c0-8  c0-8.cluster.local
+172.16.239.119 c0-9  c0-9.cluster.local
+172.16.239.120 c0-10 c0-10.cluster.local
+172.16.239.121 c0-11 c0-11.cluster.local
+172.16.239.122 c0-12 c0-12.cluster.local
+172.16.239.123 c0-13 c0-13.cluster.local
+172.16.239.124 c0-14 c0-14.cluster.local
+172.16.239.125 c0-15 c0-15.cluster.local
+172.16.239.126 c0-16 c0-16.cluster.local
+172.16.239.127 c0-17 c0-17.cluster.local
+172.16.239.128 c0-18 c0-18.cluster.local
+172.16.239.129 c0-19 c0-18.cluster.local
+172.16.239.130 c0-20 c0-18.cluster.local
 ''

rossby/login/default.nix

@@ -40,6 +40,7 @@ in
       myvnc.enable = false;
 
       os = {
+        networkmanager.enable = false;
         externalInterface = "enp65s0np0";
         nfs.enable = true;
         nfs.exports = ''
@@ -91,13 +92,13 @@ in
       #       "172.19.255.0/24"
       #   ];
         infiniband-exporter = {
-          enable = true;
+          enable = false;
           nameMap = ''
               # 0xe8ebd3030024981e "c0-1"
           '';
         };
         slurm-exporter = {
-          enable = true;
+          enable = false;
           port = 6080;
         };
       };
@@ -111,8 +112,9 @@ in
     services.flannel.iface = "enp65s0np0";
 
     networking = {
-      useDHCP = false;
+      useNetworkd = true;
       hostName = name;
+      nameservers = [ "8.8.8.8" "8.8.4.4" ];
       firewall = {
         allowedTCPPorts = [ 6443 ];
         extraCommands = ''
@@ -122,10 +124,8 @@ in
     };
 
     systemd.network = {
-      wait-online.enable = false;
+      networks."40-enp65s0np0" = {
-      networks.wired = {
         matchConfig.Name = "enp65s0np0";
-        DHCP = "no";
         address = [ "${address}/24" ];
         routes = [ { Gateway = "172.16.239.1"; } ];
       };

rossby/manage/default.nix

@@ -145,12 +145,7 @@ in {
 
     networking = {
       useNetworkd = true;
-      useDHCP = false;
       hostName = name;
-      defaultGateway = {
-        interface = "enp65s0np0";
-        address = "172.16.239.1";
-      };
       firewall = {
         allowedTCPPorts = [ 6443 4725 ];
         extraCommands = ''
@@ -167,14 +162,14 @@ in {
     };
 
     systemd.network = {
-      wait-online.enable = false;
       networks = {
-        enp65s0np0 = {
+        "40-enp65s0np0" = {
-          DHCP = "no";
           matchConfig.Name = "enp65s0np0";
+          DHCP = "no";
           address = [ "${address}/24" ];
+          routes = [ { Gateway = "172.16.239.1"; } ];
         };
-        ipoib = {
+        "45-ibp1s0" = {
           matchConfig.Name = "ibp1s0";
           DHCP = "no";
           address = [ "${ipoib}/24" ];
@@ -183,6 +178,7 @@ in {
     };
 
     services.resolved = {
+      # DNS=[::1]:53
       extraConfig = ''
         DNSStubListener=no
       '';
@@ -239,10 +235,14 @@ in {
         '';
     };
 
-    services.dnsmasq.enable = true;
+    services.coredns.enable = false;
+    services.coredns.config = {
+
+    };
+    services.dnsmasq.enable = false;
     services.dnsmasq.settings = {
-      domain = [ "cluster.local" ];
       address = [
+        "/rossby-manage.cluster.local/172.16.239.221"
         "/slurmctld.cluster.local/127.0.0.1"
       ];
       srv-host = "_slurmctld._tcp.cluster.local,slurmctld.cluster.local,6817,0,5";