fix: make hel1 obx ingress point to internal hetzber lb

modules/hpc/infiniband-exporter.nix

@@ -1,10 +1,16 @@
-{pkgs, config, lib, ...}:
+{
+  pkgs,
+  config,
+  lib,
+  ...
+}:
 with lib;
 let
   cfg = config.features.monitoring.infiniband-exporter;
 
-  python-env = pkgs.python3.withPackages (ps: with ps; [
-      prometheus_client
+  python-env = pkgs.python3.withPackages (
+    ps: with ps; [
+      prometheus-client
     ]
   );
 
@@ -26,8 +32,9 @@ let
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
       description = "Prometheus InfiniBand exporter";
-      path =  [ pkgs.rdma-core ];
-      script = "${python-env}/bin/python3 ${exporter}/infiniband-exporter.py"
+      path = [ pkgs.rdma-core ];
+      script =
+        "${python-env}/bin/python3 ${exporter}/infiniband-exporter.py"
         + " --port ${builtins.toString cfg.port} --can-reset-counter"
         + (if cfg.nameMap == null then "" else " --node-name-map=${nameMap}");
       serviceConfig = {
@@ -37,7 +44,8 @@ let
     };
   };
 
-in {
+in
+{
   options.features.monitoring.infiniband-exporter = {
     enable = mkEnableOption "Enable InfiniBand prometheus exporter";
 

obx.zone

@@ -27,24 +27,34 @@ slurm-accounting IN CNAME tos-gw.ekman.tos.obx.
 $ORIGIN vtn.obx.
 *.dev      IN CNAME rossby-manage.rossby.vtn.obx.
 *.adm      IN CNAME rossby-manage.rossby.vtn.obx.
+*.svc      IN CNAME rossby-manage.rossby.vtn.obx.
 
 $ORIGIN hel1.obx.
-gw-1         IN A 37.27.203.38
-gw-int-1     IN A 10.0.1.1
-*.dev      IN CNAME gw-1.hel1.obx.
-*.adm      IN CNAME gw-1.hel1.obx.
+gw-1             IN A 37.27.203.38
+gw-int-1         IN A 10.0.1.1
+lb-1             IN A 10.0.1.3
+controlplane-1   IN A 10.0.1.2
+controlplane-2   IN A 10.0.1.4
+controlplane-3   IN A 10.0.1.5
+ingress          IN CNAME lb-1.hel1.obx.
+*.dev            IN CNAME ingress.hel1.obx.
+*.adm            IN CNAME ingress.hel1.obx.
+*.svc            IN CNAME ingress.hel1.obx.
 
 $ORIGIN tos.obx.
 *.dev      IN CNAME ingress.k8s.tos.obx.
 *.adm      IN CNAME ingress.k8s.tos.obx.
+*.svc      IN CNAME ingress.k8s.tos.obx.
 
 $ORIGIN ekman.obx.
 *.dev      IN CNAME ekman-manage.ekman.tos.obx.
 *.adm      IN CNAME ekman-manage.ekman.tos.obx.
+*.svc      IN CNAME ekman-manage.ekman.tos.obx.
 
 $ORIGIN rossby.obx.
 *.dev      IN CNAME rossby-manage.rossby.vtn.obx.
 *.adm      IN CNAME rossby-manage.rossby.vtn.obx.
+*.svc      IN CNAME rossby-manage.rossby.vtn.obx.
 
 $ORIGIN ceph.tos.obx.
 ingress IN A 10.255.241.10

rossby/c0/default.nix

@@ -105,7 +105,7 @@ let
           hw
           ../default.nix
           ../mounts.nix
-          #./kernel.nix
+          # ./kernel.nix
         ];
       }
       // compute;

rossby/c0/kernel.nix

@@ -1,58 +1,58 @@
-{pkgs, lib, ...}:
+{ pkgs, lib, ... }:
 let
   kernel = pkgs.linuxPackages.kernel;
-  i40e =
-    pkgs.stdenv.mkDerivation rec {
-      name = "i40e-${version}-${kernel.version}";
-      version = "2.13.10";
+  i40e = pkgs.stdenv.mkDerivation rec {
+    name = "i40e-${version}-${kernel.version}";
+    version = "2.13.10";
 
-      src = pkgs.fetchFromGitHub {
-        owner = "dmarion";
-        repo = "i40e";
-        rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
-        sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
-      };
-
-      hardeningDisable = [ "pic" ];
-
-      configurePhase = ''
-        cd src
-        kernel_version=${kernel.modDirVersion}
-        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
-        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
-        export makeFlags="BUILD_KERNEL=$kernel_version"
-      '';
-
-      installPhase = ''
-        install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
-      '';
-
-      dontStrip = true;
-
-      enableParallelBuilding = true;
-
-      meta = {
-        description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
-        homepage = https://github.com/dmarion/i40e;
-        license = lib.licenses.gpl2;
-      };
+    src = pkgs.fetchFromGitHub {
+      owner = "dmarion";
+      repo = "i40e";
+      rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
+      sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
     };
+
+    hardeningDisable = [ "pic" ];
+
+    configurePhase = ''
+      cd src
+      kernel_version=${kernel.modDirVersion}
+      sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
+      sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
+      export makeFlags="BUILD_KERNEL=$kernel_version"
+    '';
+
+    installPhase = ''
+      install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
+    '';
+
+    dontStrip = true;
+
+    enableParallelBuilding = true;
+
+    meta = {
+      description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
+      homepage = "https://github.com/dmarion/i40e";
+      license = lib.licenses.gpl2;
+    };
+  };
 in
 {
   # i40e2 = i40e;
-  # boot.kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override {
-  #   argsOverride = rec {
-  #     src = pkgs.fetchurl {
-  #           url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-  #           sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid";
-  #     };
-  #     version = "5.10.239";
-  #     modDirVersion = "5.10.239";
-  #     };
-  # });
-  boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
+  boot.kernelPackages = pkgs.linuxPackagesFor (
+    pkgs.linux_5_10.override {
+      argsOverride = rec {
+        src = pkgs.fetchurl {
+          url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
+          sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid";
+        };
+        version = "5.10.239";
+        modDirVersion = "5.10.239";
+      };
+    }
+  );
+  # boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
   # overlay = self: super: {
-    # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
+  # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
   # };
 }
-

rossby/default.nix

@@ -99,7 +99,7 @@ let
       };
       cachix.enable = false;
       monitoring.nodeExporter.enable = false;
-      mft.enable = true; # Mellanox MFT
+      mft.enable = false; # Mellanox MFT
     };
 
     networking = rec {

rossby/mounts.nix

@@ -3,49 +3,81 @@ with lib;
 let
   cfg = config.cluster.mounts;
 
-  options =
-    [ "soft" "defaults" "vers=4.2" ] ++
-    (if cfg.automount.enable then [ "x-systemd.automount" ] else []);
+  options = [
+    "soft"
+    "defaults"
+    "vers=4.2"
+  ]
+  ++ (if cfg.automount.enable then [ "x-systemd.automount" ] else [ ]);
 
   users =
-    if cfg.users then {
-      "/users" = {
+    if cfg.users then
+      {
+        "/users" = {
           device = "/ceph/volumes/nfs/home";
-          options = [ "bind"  ];
-      };
-    } else {};
+          options = [
+            "bind"
+            "nofail"
+          ];
+        };
+      }
+    else
+      { };
 
   opt =
-    if cfg.opt then {
-      "/opt/bin" = {
+    if cfg.opt then
+      {
+        "/opt/bin" = {
           device = "/ceph/volumes/nfs/opt/bin";
-          options = [ "bind" ];
-      };
-      "/opt/sif" = {
+          options = [
+            "bind"
+            "nofail"
+          ];
+        };
+        "/opt/sif" = {
           device = "/ceph/volumes/nfs/opt/sif";
-          options = [ "bind" ];
-      };
-    } else {};
+          options = [
+            "bind"
+            "nofail"
+          ];
+        };
+      }
+    else
+      { };
 
   data =
-      if cfg.ceph then {
+    if cfg.ceph then
+      {
         "/data" = {
           device = "/ceph";
-          options = [ "bind" ];
+          options = [
+            "bind"
+            "nofail"
+          ];
         };
-      } else {};
+      }
+    else
+      { };
 
   work =
-      if cfg.work then {
+    if cfg.work then
+      {
+        # "/work" = {
+        #   device = "/ceph/work";
+        #   options = [ "bind" "nofail" ];
+        # };
         "/work" = {
-          device = if cfg.rdma.enable then "10.16.239.210:/work" else "172.16.239.210:/work";
+          device = "172.16.239.210:/work";
           fsType = "nfs4";
-          options =  options ++ (if cfg.rdma.enable then [ "rdma" ] else []);
+          inherit options;
         };
-    } else {};
+      }
+    else
+      { };
 
-  ceph  =
-    if cfg.ceph then {
+  ceph =
+    if cfg.ceph then
+      {
         "/ceph" = {
           device = "oceanbox@.data=/";
           fsType = "ceph";
@@ -53,46 +85,52 @@ let
             "mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789"
             "_netdev"
             "x-systemd.automount"
+            "nofail"
           ];
         };
-      } else {};
+      }
+    else
+      { };
 
   fileSystems = users // opt // data // work // ceph;
 
-  automount = mountpoint:
-      if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then
-        [{
+  automount =
+    mountpoint:
+    if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then
+      [
+        {
           wantedBy = [ "multi-user.target" ];
           automountConfig = {
             TimeoutIdleSec = "600";
           };
           where = mountpoint;
-        }]
-      else [];
+        }
+      ]
+    else
+      [ ];
 
   automounts =
-      [] ++
-      automount "/work" ++
-      automount "/opt" ++
-      automount "/backup" ++
-      automount "/data";
+    [ ] ++ automount "/work" ++ automount "/opt" ++ automount "/backup" ++ automount "/data";
 
   cephConf =
-   if cfg.ceph then {
-      "ceph/ceph.conf" = {
-        text = ''
-          [global]
-          mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789
-          log file = /tmp/ceph-$pid.log
-          [client.oceanbox]
-          key = AQDb7sZokwnUAxAANjnzxX0p+W/FUtSYryuyqg==
-          [client.rbd]
-          key = replaceme
-        '';
-        mode = "0660";
-        group = "admin";
-      };
-    } else {};
+    if cfg.ceph then
+      {
+        "ceph/ceph.conf" = {
+          text = ''
+            [global]
+            mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789
+            log file = /tmp/ceph-$pid.log
+            [client.oceanbox]
+            key = AQDb7sZokwnUAxAANjnzxX0p+W/FUtSYryuyqg==
+            [client.rbd]
+            key = replaceme
+          '';
+          mode = "0660";
+          group = "admin";
+        };
+      }
+    else
+      { };
 
 in
 {

users.nix

@@ -117,6 +117,7 @@
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas-3"
         "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst"
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik"
       ];
     };