From b839a0976cc9686dad82e9f45ab9f31bd9cf9f83 Mon Sep 17 00:00:00 2001
From: Jonas Juselius <jonas.juselius@tromso.serit.no>
Date: Thu, 7 Mar 2024 15:36:58 +0100
Subject: [PATCH] fix: split network filesystem configs

---
 cluster/ceph.nix  | 33 +++++++++++++++++++++++++++
 cluster/nfs.nix   | 24 ++++++++++++++++++++
 configuration.nix | 58 +++++++++++++++++++++++++----------------------
 modules           |  2 +-
 4 files changed, 89 insertions(+), 28 deletions(-)
 create mode 100644 cluster/ceph.nix
 create mode 100644 cluster/nfs.nix

diff --git a/cluster/ceph.nix b/cluster/ceph.nix
new file mode 100644
index 0000000..d926d03
--- /dev/null
+++ b/cluster/ceph.nix
@@ -0,0 +1,33 @@
+{
+    fileSystems = {
+      "/ceph" = {
+        device = "10.255.241.30/10.255.241.31/10.255.241.32:6789:/";
+        fsType = "ceph";
+        options = [ "name=oceanbox" "secretfile=/etc/ceph/ceph.client.oceanbox.keyring"];
+        #options = [ "name=csi-cephfs-provisioner" "secretfile=/etc/ceph/ceph.client.csi-cephfs-provisioner.keyring"];
+      };
+    };
+
+    environment.etc = {
+      "ceph/ceph.conf" = {
+        text = ''
+          [global]
+            mon_host=10.255.241.30:6789,10.255.241.31:6789,10.244.241.32:6789
+            log file = /tmp/ceph-$pid.log
+        '';
+        mode = "0644";
+      };
+      "ceph/ceph.client.oceanbox.keyring" = {
+        text = ''
+            AQDoLuhla6BWExAA6JTgxlsZkaVdxCj8GKM/UA==
+        '';
+        mode = "0644";
+      };
+      "ceph/ceph.client.csi-cephfs-provisioner.keyring" = {
+        text = ''
+           AQAkmOFlL+WeDBAAZmt4Uwtv7duMyC0MNheXuw==
+        '';
+        mode = "0644";
+      };
+    };
+}
diff --git a/cluster/nfs.nix b/cluster/nfs.nix
new file mode 100644
index 0000000..dc209a6
--- /dev/null
+++ b/cluster/nfs.nix
@@ -0,0 +1,24 @@
+{
+  fileSystems = {
+    "/opt" = {
+      device = "10.255.243.80:/opt";
+      fsType = "nfs";
+      options = [ "soft" "rdma" "defaults" "vers=4.2" ];
+    };
+    "/data" = {
+      device = "10.255.243.80:/data";
+      fsType = "nfs";
+      options = [ "soft" "rdma" "defaults" "vers=4.2" ];
+    };
+    "/backup" = {
+      device = "10.255.243.81:/backup";
+      fsType = "nfs";
+      options = [ "soft" "rdma" "defaults" "vers=4.2" ];
+    };
+    "/work" = {
+      device = "10.255.243.90:/work";
+      fsType = "nfs";
+      options = [ "soft" "rdma" "defaults" "vers=4.2" ];
+    };
+  };
+}
diff --git a/configuration.nix b/configuration.nix
index 84bc43a..ead6d21 100644
--- a/configuration.nix
+++ b/configuration.nix
@@ -11,6 +11,27 @@ let
             ipoib = "10.255.243.${toString idx}";
             pubkey = ./cluster/ekman/ekman.pub;
           }
+          rec {
+            idx = 80;
+            name = "fs0";
+            address = "10.255.241.${toString idx}";
+            ipoib = "10.255.243.${toString idx}";
+            pubkey = ./cluster/fs0/fs0.pub;
+          }
+          rec {
+            idx = 81;
+            name = "fs2";
+            address = "10.255.241.${toString idx}";
+            ipoib = "10.255.243.${toString idx}";
+            pubkey = ./cluster/fs2/fs2.pub;
+          }
+          rec {
+            idx = 90;
+            name = "fs1";
+            address = "10.255.241.${toString idx}";
+            ipoib = "10.255.243.${toString idx}";
+            pubkey = ./cluster/fs1/fs1.pub;
+          }
       ];
   etcdCluster = import ./cluster/etcdCluster.nix;
   name = "frontend";
@@ -169,7 +190,7 @@ in {
             }
         ];
       };
-      interfaces.enp175s0f0 = {
+      interfaces.eno2 = {
         useDHCP = false;
         ipv4.addresses = [
             {
@@ -191,8 +212,6 @@ in {
       firewall = {
         allowedTCPPorts = [ 4443 ];
         extraCommands = ''
-          iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
-          iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
           # needed for nodeport access on k1 and k2
           iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.241.0/24 -j SNAT --to-source 10.255.242.3
           iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
@@ -201,21 +220,6 @@ in {
     };
 
     fileSystems = {
-      "/opt" = {
-        device = "10.255.243.80:/opt";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
-      };
-      "/data" = {
-        device = "10.255.243.80:/data";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
-      };
-      "/work" = {
-        device = "10.255.243.90:/work";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
-      };
       "/exports/public" = {
         device = "/srv/public";
         options = [ "bind" ];
@@ -266,15 +270,15 @@ in {
     };
 
     system.activationScripts = {
-        # home-permissions.text = ''
-        #     chmod 755 /home/olean
-        #     chmod 755 /home/frankgaa
-        #     chmod 755 /home/jonas
-        #     chmod 755 /home/stig
-        #     chmod 755 /home/bast
-        #     chmod 755 /home/simenlk
-        #     chmod 755 /work/kraken
-        # '';
+        home-permissions.text = ''
+            chmod 755 /home/olean
+            chmod 755 /home/frankgaa
+            chmod 755 /home/jonas
+            chmod 755 /home/stig
+            chmod 755 /home/bast
+            chmod 755 /home/simenlk
+            chmod 755 /work/kraken
+        '';
     };
 
 
diff --git a/modules b/modules
index 3c35d46..6c26659 160000
--- a/modules
+++ b/modules
@@ -1 +1 @@
-Subproject commit 3c35d466b071c97af4646b1273c3d373f6d1c421
+Subproject commit 6c26659a0b39c8a2a741e5c8d83421ef3f23aaed