feat: convert rossby-manage to networkd and resolved and fix dnsmasq

rossby/manage/default.nix

@@ -7,21 +7,21 @@ let
            idx = 222;
            name = "rossby";
            address = "172.16.239.${toString idx}";
-           ipoib = "10.1.6.${toString idx}";
+           ipoib = "10.16.239.${toString idx}";
            pubkey = ../login/ssh_host_key.pub;
          }
          rec {
            idx = 210;
            name = "fs-work";
            address = "172.16.239.${toString idx}";
-           ipoib = "10.1.6.${toString idx}";
+           ipoib = "10.16.239.${toString idx}";
            pubkey = ../fs-work/ssh_host_key.pub;
          }
       ];
   etcdCluster = import ../etcdCluster.nix;
   name = "rossby-manage";
   address = "172.16.239.221";
-  ipoib = "10.1.6.221";
+  ipoib = "10.16.239.221";
 in {
     systemd.targets = {
       sleep.enable = false;
@@ -68,6 +68,7 @@ in {
 
       os = {
         externalInterface = "enp65s0np0";
+        networkmanager.enable = false;
         nfs.enable = false;
         nfs.exports = ''
           /exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
@@ -143,10 +144,13 @@ in {
     services.flannel.iface = "enp65s0np0";
 
     networking = {
-      useDHCP = false;
       useNetworkd = true;
+      useDHCP = false;
       hostName = name;
-      # defaultGateway = "172.16.239.1";
+      defaultGateway = {
+        interface = "enp65s0np0";
+        address = "172.16.239.1";
+      };
       firewall = {
         allowedTCPPorts = [ 6443 4725 ];
         extraCommands = ''
@@ -159,6 +163,7 @@ in {
           # iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j SNAT --to-source 10.255.242.3
         '';
       };
+
     };
 
     systemd.network = {
@@ -168,9 +173,11 @@ in {
           DHCP = "no";
           matchConfig.Name = "enp65s0np0";
           address = [ "${address}/24" ];
-          routes = [
-            { Gateway = "172.16.239.1"; }
-          ];
+        };
+        ipoib = {
+          matchConfig.Name = "ibp1s0";
+          DHCP = "no";
+          address = [ "${ipoib}/24" ];
         };
       };
     };
@@ -178,6 +185,9 @@ in {
     services.resolved = {
       enable = true;
       dnssec = "false";
+      extraConfig = ''
+        DNSStubListener=no
+      '';
       fallbackDns = [
         "1.1.1.1"
         "1.0.0.1"
@@ -238,11 +248,6 @@ in {
     services.dnsmasq.enable = true;
     services.dnsmasq.settings = {
       domain = [ "cluster.local" ];
-      server = [
-        "/.obx/100.100.100.100" # headscale dns
-        "8.8.8.8"
-        "1.1.1.1"
-      ];
       address = [
         "/slurmctld.cluster.local/127.0.0.1"
       ];
@@ -290,9 +295,7 @@ in {
     extraUpFlags = [
       "--login-server=https://headscale.svc.oceanbox.io"
       "--accept-dns=false" # see dnsmasq
-      "--accept-routes=true"
-      "--advertise-exit-node"
-      "--advertise-routes=172.16.239.0/24,172.16.238.0/24"
+      "--accept-routes=false"
     ];
   };
   services.networkd-dispatcher = {