wip: convert ekman to new cluster sturcture (not complete)

cluster/hosts.nix

@@ -1,83 +0,0 @@
-''
-    10.255.242.2 ekman-gw ekman-gw.compute.local ekman-gw.cluster.local
-    10.255.242.3 front-gw front-gw.compute.local front-gw.cluster.local
-
-    10.255.241.90  fs-work fs-work.compute.local
-    10.255.241.90  nfs1 nfs1.compute.local
-    10.255.241.90  fs1 fs1.compute.local
-    10.255.241.80  fs-backup fs-backup.compute.local
-    10.255.241.80  fs2 fs2.compute.local
-    10.255.241.100 ekman ekman.compute.local ekman.cluster.local
-    10.255.241.100 etcd0 etcd0.compute.local
-    10.255.241.80 etcd1 etcd1.compute.local
-    10.255.241.90 etcd2 etcd2.compute.local
-    10.255.241.99 frontend frontend.compute.local frontend.cluster.local
-    10.255.243.99 ibfrontend ibfrontend.compute.local ibfrontend.cluster.local
-
-    10.255.241.101 c0-1  c0-1.compute.local
-    10.255.241.102 c0-2  c0-2.compute.local
-    10.255.241.103 c0-3  c0-3.compute.local
-    10.255.241.104 c0-4  c0-4.compute.local
-    10.255.241.105 c0-5  c0-5.compute.local
-    10.255.241.106 c0-6  c0-6.compute.local
-    10.255.241.107 c0-7  c0-7.compute.local
-    10.255.241.108 c0-8  c0-8.compute.local
-    10.255.241.109 c0-9  c0-9.compute.local
-    10.255.241.110 c0-10 c0-10.compute.local
-    10.255.241.111 c0-11 c0-11.compute.local
-    10.255.241.112 c0-12 c0-12.compute.local
-    10.255.241.113 c0-13 c0-13.compute.local
-    10.255.241.114 c0-14 c0-14.compute.local
-    10.255.241.115 c0-15 c0-15.compute.local
-    10.255.241.116 c0-16 c0-16.compute.local
-    10.255.241.117 c0-17 c0-17.compute.local
-    10.255.241.118 c0-18 c0-18.compute.local
-
-    10.255.241.121 c1-1 c1-1.compute.local
-    10.255.241.122 c1-2 c1-2.compute.local
-    10.255.241.123 c1-3 c1-3.compute.local
-    10.255.241.124 c1-4 c1-4.compute.local
-    10.255.241.125 c1-5 c1-5.compute.local
-    10.255.241.126 c1-6 c1-6.compute.local
-    10.255.241.127 c1-7 c1-7.compute.local
-    10.255.241.128 c1-8 c1-8.compute.local
-
-    10.255.243.90  ibfs-work ibfs-work.compute.local
-    10.255.243.90  ibnfs1 ibnfs1.compute.local
-    10.255.243.90  ibfs1 ibfs1.compute.local
-    10.255.243.80  ibfs-backup ibfs-backup.compute.local
-    10.255.243.80  ibfs2 ibfs2.compute.local
-    10.255.243.100 ibekman ibekman.compute.local
-    10.255.243.100 ibetcd0 ibetcd0.compute.local
-    10.255.243.80 ibetcd1 ibetcd1.compute.local
-    10.255.243.90 ibetcd2 ibetcd2.compute.local
-
-    10.255.243.101 ib0-1  ib0-1.compute.local
-    10.255.243.102 ib0-2  ib0-2.compute.local
-    10.255.243.103 ib0-3  ib0-3.compute.local
-    10.255.243.104 ib0-4  ib0-4.compute.local
-    10.255.243.105 ib0-5  ib0-5.compute.local
-    10.255.243.106 ib0-6  ib0-6.compute.local
-    10.255.243.107 ib0-7  ib0-7.compute.local
-    10.255.243.108 ib0-8  ib0-8.compute.local
-    10.255.243.109 ib0-9  ib0-9.compute.local
-    10.255.243.110 ib0-10 ib0-10.compute.local
-    10.255.243.111 ib0-11 ib0-1.compute.local
-    10.255.243.112 ib0-12 ib0-12.compute.local
-    10.255.243.113 ib0-13 ib0-13.compute.local
-    10.255.243.114 ib0-14 ib0-14.compute.local
-    10.255.243.115 ib0-15 ib0-15.compute.local
-    10.255.243.116 ib0-16 ib0-16.compute.local
-    10.255.243.117 ib0-17 ib0-17.compute.local
-    10.255.243.118 ib0-18 ib0-18.compute.local
-    10.255.243.118 ib0-18 ib0-19.compute.local
-
-    10.255.243.121 ib1-1  ib1-1.compute.local
-    10.255.243.122 ib1-2  ib1-2.compute.local
-    10.255.243.123 ib1-3  ib1-3.compute.local
-    10.255.243.124 ib1-4  ib1-4.compute.local
-    10.255.243.125 ib1-5  ib1-5.compute.local
-    10.255.243.126 ib1-6  ib1-6.compute.local
-    10.255.243.127 ib1-7  ib1-7.compute.local
-    10.255.243.128 ib1-8  ib1-8.compute.local
-''

ekman.nix

@@ -0,0 +1,25 @@
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
+  # }) {};
+  pkgs = import <nixpkgs> {};
+
+  ekman-manage = {
+      deployment = {
+          tags = [ "manage" "ekman" ];
+          allowLocalDeployment = true;
+          targetHost = null;
+      };
+      imports = [ ./ekman/manage ];
+  };
+
+  ekman-login = import ./ekman/login { inherit pkgs; };
+  c0 = import ./ekman/c0 { inherit pkgs; };
+  c0x = import ./ekman/c0x { inherit pkgs; };
+  c1 = import ./ekman/c1 { inherit pkgs; };
+  fs-work = import ./ekman/fs-work { inherit pkgs; };
+  fs-backup = import ./ekman/fs-backup { inherit pkgs; };
+in
+    { inherit ekman-manage; } // ekman-login // c0 // c0x // c1 // fs-work // fs-backup

ekman/c0/default.nix

@@ -35,7 +35,7 @@ let
 
   mkCompute = host:
   let
-    hw = ./hw + "/${host.name}.nix";
+    hw = ./hardware-configuration.d + "/${host.name}.nix";
   in {
     "${host.name}" = {
       cluster = {

ekman/c0/nodes.nix

@@ -7,5 +7,5 @@ map (n: (
     name = "c0-${toString n}";
     address = "10.255.241.${toString idx}";
     ipoib = "10.255.243.${toString idx}";
-    pubkey = ./. +  "/pubkeys/c0-${toString n}.pub";
+    pubkey = ./. +  "/ssh_host_key.d/c0-${toString n}.pub";
   })) nodes

ekman/c0x/default.nix

@@ -35,7 +35,7 @@ let
 
   mkCompute = host:
   let
-    hw = ./hw + "/${host.name}.nix";
+    hw = ./hardware-configuration.d + "/${host.name}.nix";
   in {
     "${host.name}" = {
       cluster = {

ekman/c0x/nodes.nix

@@ -7,5 +7,5 @@ map (n: (
     name = "c0-${toString n}";
     address = "10.255.241.${toString idx}";
     ipoib = "10.255.243.${toString idx}";
-    pubkey = ./. +  "/pubkeys/c0-${toString n}.pub";
+    pubkey = ./. +  "/ssh_host_key.d/c0-${toString n}.pub";
   })) nodes

ekman/c1/default.nix

@@ -21,7 +21,7 @@ let
 
   mkCompute = host:
   let
-    hw = ./hw + "/${host.name}.nix";
+    hw = ./hardware-configuration.d + "/${host.name}.nix";
   in {
     "${host.name}" = {
       cluster = {

ekman/c1/nodes.nix

@@ -8,5 +8,5 @@ map (n: (
     address = "10.255.241.${toString idx}";
     ipoib = "10.255.243.${toString idx}";
     gbe100 = "10.255.244.${toString idx}";
-    pubkey = ./. +  "/pubkeys/c1-${toString n}.pub";
+    pubkey = ./. +  "/ssh_host_key.d/c1-${toString n}.pub";
   })) nodes

ekman/default.nix

@@ -100,7 +100,7 @@ let
     };
 
     networking = {
-      domain = mkDefault "compute.local";
+      domain = mkDefault "cluster.local";
       defaultGateway = mkDefault "10.255.241.1";
       nameservers = mkDefault [ "8.8.8.8" ];
       search = mkDefault [];
@@ -146,13 +146,13 @@ let
         mungeKey = ./munge.key;
         mungeUid = mkDefault 996; # hack
         # pkey = "0x7666";
-        controlMachine = "frontend";
+        controlMachine = "ekman-manage";
         mailDomain = "oceanbox.io";
         nodeName = [
           "c0-[1-18] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
           "c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
           "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
-          "frontend Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN"
+          "ekman-manage Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN"
         ];
         partitionName = [
           "batch Nodes=c0-[1-18] Default=YES MaxTime=INFINITE State=UP"
@@ -200,7 +200,7 @@ let
         initca = ./ca;
         cidr = "10.100.0.0/16";
         master = {
-          name = "frontend";
+          name = "ekman-manage";
           address = "10.255.241.99";
           # extraSANs = [
           #     "frontend.oceanbox.io"
@@ -240,29 +240,29 @@ let
     };
 
     programs.ssh.knownHosts = {
-      frontend = {
+      ekman-manage = {
         hostNames = [
-          "frontend" "frontend.compute.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.99"
+          "ekman-manage" "ekman-manage.cluster.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.99"
         ];
-        publicKeyFile = ../frontend.pub;
+        publicKeyFile = ./manage/ssh_host_key.pub;
       };
       ekman = {
         hostNames = [
-          "ekman" "ekman.compute.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100"
+          "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100"
         ];
-        publicKeyFile = ./ekman/ekman.pub;
+        publicKeyFile = ./logon/ssh_host_key.pub;
       };
       fs-work = {
         hostNames = [
-          "fs-work" "fs-work.compute.local" "10.255.241.90" "10.255.243.90"
+          "fs-work" "fs-work.cluster.local" "10.255.241.90" "10.255.243.90"
         ];
-        publicKeyFile = ./fs-work/fs-work.pub;
+        publicKeyFile = ./fs-work/ssh_host_key.pub;
       };
       fs-backup = {
         hostNames = [
-          "fs-backup" "fs-backup.compute.local" "10.255.241.80" "10.255.243.80"
+          "fs-backup" "fs-backup.cluster.local" "10.255.241.80" "10.255.243.80"
         ];
-        publicKeyFile = ./fs-backup/fs-backup.pub;
+        publicKeyFile = ./fs-backup/ssh_host_key.pub;
       };
   } // builtins.foldl' (a: x:
       let n = toString x.idx;
@@ -270,7 +270,7 @@ let
           "${x.name}" = {
               hostNames = [
                   "${x.name}"
-                  "${x.name}.compute.local"
+                  "${x.name}.cluster.local"
                   "10.255.241.${n}"
                   "10.255.243.${n}"
               ];
@@ -324,7 +324,7 @@ in {
   imports = [
     ../modules
     ../nixos
-    ./users.nix
+    ../users.nix
   ];
 }
 

ekman/hosts.nix

@@ -0,0 +1,83 @@
+''
+    10.255.242.2 ekman-gw ekman-gw.cluster.local ekman-gw.cluster.local
+    10.255.242.3 front-gw front-gw.cluster.local front-gw.cluster.local
+
+    10.255.241.90  fs-work fs-work.cluster.local
+    10.255.241.90  nfs1 nfs1.cluster.local
+    10.255.241.90  fs1 fs1.cluster.local
+    10.255.241.80  fs-backup fs-backup.cluster.local
+    10.255.241.80  fs2 fs2.cluster.local
+    10.255.241.100 ekman ekman.cluster.local ekman.cluster.local
+    10.255.241.100 etcd0 etcd0.cluster.local
+    10.255.241.80 etcd1 etcd1.cluster.local
+    10.255.241.90 etcd2 etcd2.cluster.local
+    10.255.241.99 ekman-manage frontend frontend.cluster.local frontend.cluster.local
+    10.255.243.99 ibfrontend ibfrontend.cluster.local ibfrontend.cluster.local
+
+    10.255.241.101 c0-1  c0-1.cluster.local
+    10.255.241.102 c0-2  c0-2.cluster.local
+    10.255.241.103 c0-3  c0-3.cluster.local
+    10.255.241.104 c0-4  c0-4.cluster.local
+    10.255.241.105 c0-5  c0-5.cluster.local
+    10.255.241.106 c0-6  c0-6.cluster.local
+    10.255.241.107 c0-7  c0-7.cluster.local
+    10.255.241.108 c0-8  c0-8.cluster.local
+    10.255.241.109 c0-9  c0-9.cluster.local
+    10.255.241.110 c0-10 c0-10.cluster.local
+    10.255.241.111 c0-11 c0-11.cluster.local
+    10.255.241.112 c0-12 c0-12.cluster.local
+    10.255.241.113 c0-13 c0-13.cluster.local
+    10.255.241.114 c0-14 c0-14.cluster.local
+    10.255.241.115 c0-15 c0-15.cluster.local
+    10.255.241.116 c0-16 c0-16.cluster.local
+    10.255.241.117 c0-17 c0-17.cluster.local
+    10.255.241.118 c0-18 c0-18.cluster.local
+
+    10.255.241.121 c1-1 c1-1.cluster.local
+    10.255.241.122 c1-2 c1-2.cluster.local
+    10.255.241.123 c1-3 c1-3.cluster.local
+    10.255.241.124 c1-4 c1-4.cluster.local
+    10.255.241.125 c1-5 c1-5.cluster.local
+    10.255.241.126 c1-6 c1-6.cluster.local
+    10.255.241.127 c1-7 c1-7.cluster.local
+    10.255.241.128 c1-8 c1-8.cluster.local
+
+    10.255.243.90  ibfs-work ibfs-work.cluster.local
+    10.255.243.90  ibnfs1 ibnfs1.cluster.local
+    10.255.243.90  ibfs1 ibfs1.cluster.local
+    10.255.243.80  ibfs-backup ibfs-backup.cluster.local
+    10.255.243.80  ibfs2 ibfs2.cluster.local
+    10.255.243.100 ibekman ibekman.cluster.local
+    10.255.243.100 ibetcd0 ibetcd0.cluster.local
+    10.255.243.80 ibetcd1 ibetcd1.cluster.local
+    10.255.243.90 ibetcd2 ibetcd2.cluster.local
+
+    10.255.243.101 ib0-1  ib0-1.cluster.local
+    10.255.243.102 ib0-2  ib0-2.cluster.local
+    10.255.243.103 ib0-3  ib0-3.cluster.local
+    10.255.243.104 ib0-4  ib0-4.cluster.local
+    10.255.243.105 ib0-5  ib0-5.cluster.local
+    10.255.243.106 ib0-6  ib0-6.cluster.local
+    10.255.243.107 ib0-7  ib0-7.cluster.local
+    10.255.243.108 ib0-8  ib0-8.cluster.local
+    10.255.243.109 ib0-9  ib0-9.cluster.local
+    10.255.243.110 ib0-10 ib0-10.cluster.local
+    10.255.243.111 ib0-11 ib0-1.cluster.local
+    10.255.243.112 ib0-12 ib0-12.cluster.local
+    10.255.243.113 ib0-13 ib0-13.cluster.local
+    10.255.243.114 ib0-14 ib0-14.cluster.local
+    10.255.243.115 ib0-15 ib0-15.cluster.local
+    10.255.243.116 ib0-16 ib0-16.cluster.local
+    10.255.243.117 ib0-17 ib0-17.cluster.local
+    10.255.243.118 ib0-18 ib0-18.cluster.local
+    10.255.243.118 ib0-18 ib0-19.cluster.local
+
+    10.255.243.121 ib1-1  ib1-1.cluster.local
+    10.255.243.122 ib1-2  ib1-2.cluster.local
+    10.255.243.123 ib1-3  ib1-3.cluster.local
+    10.255.243.124 ib1-4  ib1-4.cluster.local
+    10.255.243.125 ib1-5  ib1-5.cluster.local
+    10.255.243.126 ib1-6  ib1-6.cluster.local
+    10.255.243.127 ib1-7  ib1-7.cluster.local
+    10.255.243.128 ib1-8  ib1-8.cluster.local
+''

ekman/login/default.nix

@@ -33,8 +33,8 @@ in
 
     features = {
       host = {
-        name = "ekman";
-        address = "10.255.241.100";
+        inherit name;
+        inherit address;
       };
 
       myvnc.enable = false;
@@ -133,11 +133,11 @@ in
 
     networking = {
       useDHCP = false;
-      hostName = "ekman";
+      hostName = name;
       interfaces.enp33s0f3np3 = {
         useDHCP = false;
         ipv4.addresses = [ {
-          address = "10.255.241.100";
+          address = address;
           prefixLength = 24;
         } ];
         # ipv4.routes = [

ekman/manage/default.nix

@@ -1,33 +1,33 @@
 { pkgs, ...}:
 let
   computeNodes =
-      import ./cluster/c0/nodes.nix ++
-      import ./cluster/c1/nodes.nix ++
+      import ../c0/nodes.nix ++
+      import ../c1/nodes.nix ++
       [
           rec {
             idx = 100;
             name = "ekman";
             address = "10.255.241.${toString idx}";
             ipoib = "10.255.243.${toString idx}";
-            pubkey = ./cluster/ekman/ekman.pub;
+            pubkey = ../login/ekman.pub;
           }
           rec {
             idx = 90;
             name = "fs-work";
             address = "10.255.241.${toString idx}";
             ipoib = "10.255.243.${toString idx}";
-            pubkey = ./cluster/fs-work/fs-work.pub;
+            pubkey = ../fs-work/fs-work.pub;
           }
           rec {
             idx = 81;
             name = "fs-backup";
             address = "10.255.241.${toString idx}";
             ipoib = "10.255.243.${toString idx}";
-            pubkey = ./cluster/fs-backup/fs-backup.pub;
+            pubkey = ../fs-backup/fs-backup.pub;
           }
       ];
-  etcdCluster = import ./cluster/etcdCluster.nix;
-  name = "frontend";
+  etcdCluster = import ../etcdCluster.nix;
+  name = "ekman-manage";
   address = "10.255.241.99";
   ipoib = "10.255.243.99";
 in {
@@ -102,7 +102,7 @@ in {
         server = {
           enable = false;
           scrapeHosts = [
-              "frontend"
+              "ekman-manage"
               "ekman"
               "fs-work"
               "fs-backup"
@@ -134,7 +134,7 @@ in {
           enable = true;
           nameMap = ''
             0xe8ebd3030024a2c6 "ekman"
-            0x0c42a10300ddc4bc "frontend"
+            0x0c42a10300ddc4bc "ekman-manage"
             0xe8ebd3030024a2ae "fs-work"
             0x1c34da0300787798 "fs-backup"
             0xe8ebd3030024981e "c0-1"
@@ -341,9 +341,9 @@ in {
 
   imports = [
       ./hardware-configuration.nix
-      ./cluster/cluster.nix
-      ./cluster/mounts.nix
-      ./cluster/myvnc.nix
+      ../default.nix
+      ../mounts.nix
+      ../myvnc.nix
   ];
 }
 

ekman/mounts.nix

@@ -12,8 +12,8 @@ let
     [ "soft" "defaults" "vers=4.2" ] ++
     (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []);
 
-  home =
-    if cfg.home then {
+  users =
+    if cfg.users then {
       "/frontend" = {
         device = "10.255.241.100:/home";
         fsType = "nfs4";
@@ -96,7 +96,7 @@ let
         };
       } else {};
 
-  fileSystems = home // opt // data // work // backup // ceph;
+  fileSystems = users // opt // data // work // backup // ceph;
 
   automount = mountpoint:
       if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then
@@ -139,7 +139,7 @@ in
     rdma.enable = mkEnableOption "Enable NFS over RDMA";
     gbe100.enable = mkEnableOption "Enable NFS over 100 GbE";
     automount.enable = mkEnableOption "Enable NFS automounting";
-    home = mkEnableOption "Enable /home";
+    users = mkEnableOption "Enable /users";
     opt = mkEnableOption "Enable /opt";
     data = mkEnableOption "Enable /data";
     work = mkEnableOption "Enable /work";

hive.nix

@@ -1,25 +0,0 @@
-let
-  # Pin the deployment package-set to a specific version of nixpkgs
-  # pkgs = import (builtins.fetchTarball {
-  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
-  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
-  # }) {};
-  pkgs = import <nixpkgs> {};
-
-  frontend = {
-      deployment = {
-          tags = [ "manage" "frontend" ];
-          allowLocalDeployment = true;
-          targetHost = null;
-      };
-      imports = [ ./configuration.nix ];
-  };
-
-  ekman = import ./cluster/ekman { inherit pkgs; };
-  c0 = import ./cluster/c0 { inherit pkgs; };
-  c0x = import ./cluster/c0x { inherit pkgs; };
-  c1 = import ./cluster/c1 { inherit pkgs; };
-  fs-work = import ./cluster/fs-work { inherit pkgs; };
-  fs-backup = import ./cluster/fs-backup { inherit pkgs; };
-in
-    { inherit frontend; stokes = frontend; } // ekman // c0 // c0x // c1 // fs-work // fs-backup