diff --git a/nixops/bin/adduser.sh b/cluster/bin/adduser.sh similarity index 100% rename from nixops/bin/adduser.sh rename to cluster/bin/adduser.sh diff --git a/nixops/bin/copy-hardware-configuration.sh b/cluster/bin/copy-hardware-configuration.sh similarity index 100% rename from nixops/bin/copy-hardware-configuration.sh rename to cluster/bin/copy-hardware-configuration.sh diff --git a/nixops/bin/deploy.sh b/cluster/bin/deploy.sh similarity index 100% rename from nixops/bin/deploy.sh rename to cluster/bin/deploy.sh diff --git a/nixops/bin/initca.sh b/cluster/bin/initca.sh similarity index 100% rename from nixops/bin/initca.sh rename to cluster/bin/initca.sh diff --git a/nixops/bin/reboot.sh b/cluster/bin/reboot.sh similarity index 100% rename from nixops/bin/reboot.sh rename to cluster/bin/reboot.sh diff --git a/nixops/bin/ssh.sh b/cluster/bin/ssh.sh similarity index 100% rename from nixops/bin/ssh.sh rename to cluster/bin/ssh.sh diff --git a/nixops/bin/teardown.sh b/cluster/bin/teardown.sh similarity index 100% rename from nixops/bin/teardown.sh rename to cluster/bin/teardown.sh diff --git a/nixops/ekman/cluster.nix b/cluster/cluster.nix similarity index 84% rename from nixops/ekman/cluster.nix rename to cluster/cluster.nix index 23b0020..cb5070b 100644 --- a/nixops/ekman/cluster.nix +++ b/cluster/cluster.nix @@ -3,18 +3,7 @@ with lib; let cfg = config.features.host; - nodes = import ./nodes.nix; - - etcdCluster = { - enable = true; - existing = true; - nodes = - { - etcd0 = "10.255.241.100"; - etcd1 = "10.255.241.90"; - etcd2 = "10.255.241.80"; - }; - }; + computeNodes = import ./compute/nodes.nix; mkSANs = host: [ host.name @@ -40,10 +29,6 @@ let # ]; }; - # services.udev.extraRules = '' - # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x3666" - # ''; - console = { font = "Lat2-Terminus16"; keyMap = "us"; @@ -105,7 +90,6 @@ let }; cachix.enable = false; monitoring.nodeExporter.enable = false; - pki = { ca = ./ca; }; }; networking = { @@ -123,12 +107,12 @@ let # systemd.services."serial-getty@ttyS0".enable = true; - environment.etc."beegfs/connauthfile" = { - source = ./connauthfile; - mode = "0400"; - uid = 0; - gid = 0; - }; + # environment.etc."beegfs/connauthfile" = { + # source = ./connauthfile; + # mode = "0400"; + # uid = 0; + # gid = 0; + # }; nix = { maxJobs = 32; @@ -151,26 +135,27 @@ let enable = true; slurm = { client = true; - mungeKey = ./munge.key; + mungeKey = ./compute/munge.key; mungeUid = 996; # hack + pkey = "0x7666"; controlMachine = "ekman"; nodeName = [ - "c0-[1-2] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" ]; partitionName = [ - "batch Nodes=c0-[1-2] Default=YES MaxTime=INFINITE State=UP" + "batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP" "frontend Nodes=ekman MaxTime=1:00:00 State=UP" ]; }; beegfs = { - enable = true; + enable = false; beegfs = { - beegfs0 = { + work = { mgmtdHost = "ibbeegfs0"; connAuthFile = "/etc/beegfs/connauthfile"; client = { - enable = true; + enable = false; mountPoint = "/work"; }; }; @@ -190,6 +175,11 @@ let fsType = "nfs"; options = [ "soft" "rdma" "defaults" "vers=4.2" ]; }; + "/work" = { + device = "10.255.243.90:/work"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; + }; }; }; @@ -199,7 +189,6 @@ let enable = true; node.enable = true; clusterName = "ekman"; - inherit etcdCluster; initca = ./ca; cidr = "10.100.0.0/16"; master = { @@ -210,9 +199,9 @@ let ingressNodes = [ "ekman.oceanbox.io" ]; - fileserver = "mds1-0"; + fileserver = "ibnfs0"; charts = { - acme_email = "innovasjon@itpartner.no"; + acme_email = "acme@oceanbox.io"; # grafana_smtp_user = "utvikling"; # grafana_smtp_password = "S0m3rp0m@de#21!"; }; @@ -233,15 +222,27 @@ let text = '' 10.255.241.80 10.255.241.90 - '' + builtins.foldl' (a: x: a + "${x.address}\n") "" nodes; + '' + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes; }; programs.ssh.knownHosts = { ekman = { hostNames = [ - "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.8" + "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100" ]; - publicKeyFile = ./pubkeys/ekman.pub; + publicKeyFile = ../ekman.pub; + }; + nfs0 = { + hostNames = [ + "nfs0" "nfs0.cluster.local" "10.255.241.80" "10.255.243.80" + ]; + publicKeyFile = ./nfs0/nfs0.pub; + }; + nfs1 = { + hostNames = [ + "nfs1" "nfs1.cluster.local" "10.255.241.90" "10.255.243.90" + ]; + publicKeyFile = ./nfs1/nfs1.pub; }; } // builtins.foldl' (a: x: let @@ -255,9 +256,9 @@ let "10.255.241.${addr}" "10.255.243.${addr}" ]; - publicKeyFile = ./pubkeys/c0-${n}.pub; + publicKeyFile = ./compute/pubkeys/c0-${n}.pub; }; - }) {} nodes; + }) {} computeNodes; environment.systemPackages = [ openssh-shosts ]; @@ -293,8 +294,8 @@ in { ]; imports = [ - ../../modules - ../../nixos + ../modules + ../nixos ./users.nix ]; } diff --git a/nixops/ekman/connauthfile b/cluster/compute/connauthfile similarity index 100% rename from nixops/ekman/connauthfile rename to cluster/compute/connauthfile diff --git a/nixops/ekman/default.nix b/cluster/compute/default.nix similarity index 78% rename from nixops/ekman/default.nix rename to cluster/compute/default.nix index 9c9695b..a93e18e 100644 --- a/nixops/ekman/default.nix +++ b/cluster/compute/default.nix @@ -1,15 +1,16 @@ +{ pkgs ? import {} }: let # Pin the deployment package-set to a specific version of nixpkgs # pkgs = import (builtins.fetchTarball { # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; # }) {}; - pkgs = import {}; + # pkgs = import {}; nodes = import ./nodes.nix; compute = { - # deployment.tags = [ "compute" ]; + deployment.tags = [ "compute" ]; fileSystems = { "/frontend" = { @@ -55,11 +56,12 @@ let deployment.targetHost = host.address; - # services.udev.extraRules = '' - # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222" - # ''; + services.udev.extraRules = '' + KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + ''; networking = { + hostName = host.name; useDHCP = false; interfaces.enp33s0f0np0 = { useDHCP = false; @@ -74,7 +76,7 @@ let } ]; }; - interfaces.ibp1s0 = { + interfaces."ibp1s0.7666" = { useDHCP = false; ipv4.addresses = [ { address = ipoib; @@ -82,21 +84,9 @@ let } ]; }; }; - imports = [ ./cluster.nix hw ]; + imports = [ ../cluster.nix hw ]; } // compute; }; in builtins.foldl' (a: n: a // mkCompute n) {} nodes -#{ - ## morph - # network = { - # inherit pkgs; - # description = "ekman"; - # ordering = { - # tags = [ "frontend" "compute" ]; - # }; - # }; - - # inherit ekman; -# } // builtins.foldl' (a: n: a // mkCompute n) {} nodes diff --git a/nixops/ekman/hw/c0-1.nix b/cluster/compute/hw/c0-1.nix similarity index 100% rename from nixops/ekman/hw/c0-1.nix rename to cluster/compute/hw/c0-1.nix diff --git a/cluster/compute/hw/c0-10.nix b/cluster/compute/hw/c0-10.nix new file mode 100644 index 0000000..6c8d140 --- /dev/null +++ b/cluster/compute/hw/c0-10.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f4579aea-21f4-4d4f-9c65-53f9b540cac4"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/A20A-720E"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-11.nix b/cluster/compute/hw/c0-11.nix new file mode 100644 index 0000000..8eb54b3 --- /dev/null +++ b/cluster/compute/hw/c0-11.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/30add79f-20c1-4030-8d72-47a3dc6b533d"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/716E-5AC4"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-12.nix b/cluster/compute/hw/c0-12.nix new file mode 100644 index 0000000..0830e27 --- /dev/null +++ b/cluster/compute/hw/c0-12.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/26d04544-c68a-4e7a-a728-5676f472ebb7"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/7E82-333A"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-13.nix b/cluster/compute/hw/c0-13.nix new file mode 100644 index 0000000..ffd8eaa --- /dev/null +++ b/cluster/compute/hw/c0-13.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/a3fd9989-866c-4d24-9c59-7c2803a28eae"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/8B58-FF6B"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-14.nix b/cluster/compute/hw/c0-14.nix new file mode 100644 index 0000000..3357395 --- /dev/null +++ b/cluster/compute/hw/c0-14.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/4db39a06-d119-4efb-9111-41c85e622467"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/B8A4-E5C5"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-15.nix b/cluster/compute/hw/c0-15.nix new file mode 100644 index 0000000..91d3467 --- /dev/null +++ b/cluster/compute/hw/c0-15.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/cc8f9ec9-fe2c-4603-8ba1-2df9c8389cbf"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/B26B-7B6E"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-16.nix b/cluster/compute/hw/c0-16.nix new file mode 100644 index 0000000..a7a645d --- /dev/null +++ b/cluster/compute/hw/c0-16.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/6923deaa-ab2e-45c1-bfe6-7fe8be6f1770"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/C9D4-6AE3"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/nixops/ekman/hw/c0-2.nix b/cluster/compute/hw/c0-2.nix similarity index 100% rename from nixops/ekman/hw/c0-2.nix rename to cluster/compute/hw/c0-2.nix diff --git a/cluster/compute/hw/c0-3.nix b/cluster/compute/hw/c0-3.nix new file mode 100644 index 0000000..e63952d --- /dev/null +++ b/cluster/compute/hw/c0-3.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/db0635c5-e0bf-4d85-9834-f0374f8ffa9c"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/A3AE-176D"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-4.nix b/cluster/compute/hw/c0-4.nix new file mode 100644 index 0000000..7e29d0e --- /dev/null +++ b/cluster/compute/hw/c0-4.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/886d3243-07c5-4679-9979-cef11a9b5cb1"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/6FA3-B1DE"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-5.nix b/cluster/compute/hw/c0-5.nix new file mode 100644 index 0000000..72452cc --- /dev/null +++ b/cluster/compute/hw/c0-5.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/aa46e116-30eb-4e49-996b-7ee49fb1914c"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/8DF7-E9B8"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-6.nix b/cluster/compute/hw/c0-6.nix new file mode 100644 index 0000000..3514f6a --- /dev/null +++ b/cluster/compute/hw/c0-6.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f972da1f-a7da-4c94-8bee-cf733a3e4aef"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/7031-4731"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-7.nix b/cluster/compute/hw/c0-7.nix new file mode 100644 index 0000000..2fa085c --- /dev/null +++ b/cluster/compute/hw/c0-7.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/de9abab9-3280-4ef2-b467-ea09842b1be6"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/63E2-0C5A"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-8.nix b/cluster/compute/hw/c0-8.nix new file mode 100644 index 0000000..d541207 --- /dev/null +++ b/cluster/compute/hw/c0-8.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/22b9bf44-cafb-4f06-a23f-9b6f4a748e81"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/5CBE-8643"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/compute/hw/c0-9.nix b/cluster/compute/hw/c0-9.nix new file mode 100644 index 0000000..62df049 --- /dev/null +++ b/cluster/compute/hw/c0-9.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/4a8e1c95-8bbb-40ed-8865-74bab73e4ee8"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/7D28-3770"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/nixops/ekman/kernel.nix b/cluster/compute/kernel.nix similarity index 100% rename from nixops/ekman/kernel.nix rename to cluster/compute/kernel.nix diff --git a/nixops/ekman/munge.key b/cluster/compute/munge.key similarity index 100% rename from nixops/ekman/munge.key rename to cluster/compute/munge.key diff --git a/nixops/ekman/myvnc.nix b/cluster/compute/myvnc.nix similarity index 100% rename from nixops/ekman/myvnc.nix rename to cluster/compute/myvnc.nix diff --git a/nixops/ekman/nodes.nix b/cluster/compute/nodes.nix similarity index 75% rename from nixops/ekman/nodes.nix rename to cluster/compute/nodes.nix index 28dca4f..01b0128 100644 --- a/nixops/ekman/nodes.nix +++ b/cluster/compute/nodes.nix @@ -1,3 +1,3 @@ with builtins; -let nodes = genList (n: n + 1) 2; in +let nodes = genList (n: n + 1) 16; in map (n: ({ idx = n; name = "c0-${toString n}"; address = "10.255.241.${toString (n + 100)}"; })) nodes diff --git a/cluster/compute/pubkeys/beegfs0.pub b/cluster/compute/pubkeys/beegfs0.pub new file mode 100644 index 0000000..4513099 --- /dev/null +++ b/cluster/compute/pubkeys/beegfs0.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCDZcZAPsYuC9UKS2Eme1x79sumjcPMduQ+B1K9zvyU5MWGOmuEIaNoFB2XNUFxjtXVTU9gy/33jiSQvLJdJ7yrSyyfVBREZJrFXONvHy4jS5xNGrS5gc1ih+5n3lMBxiCrP4OeI1sXCKDNIBOvRlsRFB0Sj90NbgtDIHjyd+OKJ9mCPdMe+cnSdELZH1jO8NjpUckdqh/PeOX28XQho1VrpJO4HlsR4id3yONycS5cZT4Sb2kW/3SwUa3TLmgDTijJl+ugOUCbd6d5H6JX14/Xq4rCejrHATAmCjmuj4FLI399kZBo4Ymp1KLJVTnCuFxLpMBfeVu0BL7XBebFhfYt7P3fZIUf/73YZ6fj6b81KdQOpKoi/lC6h3IBmf4YhYVSjAJJC0UH5sES/W7Hl46Y4+7Tpo7SrTT/zBHDr1GrQWEfnwYdjXlgu9UzF3S1lKJeVMalKLnn1lp/ZdfnjrukD64znv59VrRgBNBWEbGUa7DNyUx68No8FlEhggMX+E= root@beegfs0 diff --git a/nixops/ekman/pubkeys/c0-1.pub b/cluster/compute/pubkeys/c0-1.pub similarity index 100% rename from nixops/ekman/pubkeys/c0-1.pub rename to cluster/compute/pubkeys/c0-1.pub diff --git a/cluster/compute/pubkeys/c0-10.pub b/cluster/compute/pubkeys/c0-10.pub new file mode 100644 index 0000000..0808800 --- /dev/null +++ b/cluster/compute/pubkeys/c0-10.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCkogj7O965v6DYNCld411vyUpsAL8+6hbVT2fjC1RZVyR01AW/+Mvle6aM90XpCMgwUCxgZIULNG4waFlCxXt5IGbOO3/vkHdznK9TUNBJllg69KAdp9nAePyZIczzxRXGh5lzu9RAE7Dywz8hkgELwHokGq8BBQ620/prMdlj4GRsSk3A3hdvbC5XuhwL+KJjrSecELjcvDtiURKDgkDglg7b3bahVais0meaMjkfYL4w1nm4AHnwDlriP6y3hJwBH3odJ1MdJmZBnS0Ebf/wYl2vL0kNACaQGh1+gzdhw2zviv7rW9Wuir07h6CuAQ1SV/b92lzk2Zekaeixf/6I1YNHWiQhfEVOZG/zibm8LdidJxUoLCxyO0h2nBGI3iuPgzUzAmrDSG8nSGNMJw3GU6ub4nj8s+6ZxV7nZ8omozqm6CP9Dw4H4iSRF1CVS3ePH44Zssgd7sbaPG5w4NAniKLRiTTwFfLYyyAhm7APHNrunFPX9SD2H2rJiqJ+16e4rDiBFjoQz5AO/+za+ITSNWZX7BLZM2Swo7jvrZWXYQQGhRj9r3Lzw6zZol+waC+uU4i5JBvQk4ksSNWkTAl/i2GpiUdy94ksBRSZCUWA6gQ49M8iftFNsLsT4KkuIpgV06zHSHKs/9NNovwiINiQo+zGBNNy0eGZTOXHIV/6nw== root@nixos diff --git a/cluster/compute/pubkeys/c0-11.pub b/cluster/compute/pubkeys/c0-11.pub new file mode 100644 index 0000000..2a2edd6 --- /dev/null +++ b/cluster/compute/pubkeys/c0-11.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC9X4NFHsCryXjQOiUrxcTNOx0JpPiGViEuvEhB23C0270bwbxlijaIsTrAxB8lfOaKHGvzXfMBX0lV9QGXfIPTXIr0hbmOy6xfJq6lRQwbiRA6YS4T1yic25mx47F0ODqaW3mTykpIE1qpz4Wzl3M1ZLbbT1wcvWTFJ6MCY74OR1vNrtR7FgVFrh7q7iT1+a8Qzr5zG39yphO9aYiBKNLle820oH1UxsJ3W5e38ZI/huJ2w1Y0TY6eghedEZvGIhm0qDJHlc8EmGterCUjCLZKcZgHNGIC6FnT3n8HGwsTtYHc2mRW3W4BK1iIpncYkDSX71xY4ev13Y7UbD6rVBWqC+eKD/3pm+HlvXxk8tvDZfcq4ngSEIoU7d5FeLgE2hidE92zLaKdXpJ0PaLo1Gz3M0CY3TeaCWZBSnToGY106ynUipoGJ29cDaQWVlYkWAdC7d+3eR1yjpT/98PNx08m8fVs/PT30/LkR6kpUpCB1GBdOu1/Q4DFeGtY1BFNibzoWyk6ByagOMdEfRzPn99J4Vd0zlzwyzFkJTlI7OHeVwBFPHW0joGm3x3MGbRmFQBBIA3C9b9a6mFseQpMX/T++v0h98x7mvsMx2Nf+1O3uRx3yLp47GfyatsKB8oUSi5dI7S3cAyAZIB8E8ot4dkdounPzA2L8ipnzwdEjpqwLw== root@nixos diff --git a/cluster/compute/pubkeys/c0-12.pub b/cluster/compute/pubkeys/c0-12.pub new file mode 100644 index 0000000..b012c4e --- /dev/null +++ b/cluster/compute/pubkeys/c0-12.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+VIsgEyusSjMuU26gQAYNoWs9iC1ErHVjmZNpbtzZ5IlR4RdsmsStf1QZUSG5FJRA2pMOaZD0QUOFI4t4k4VWsYwVXhgkcjF1u0+nlc8XMSihmIHsNPZNYBVRyuL4shegjezKn4EZoHn41UxAcpWAg9XDIRsENAm1G8RcPqwmhrLg4KpC482iuqJT+GavdEcV5rT0O3wCeq0EyFpyd8wX3JARKiyBA5HjyiVzJTeycda3ffudwDmsym4JQeYHretj5REc1Wn4qOVrn6m/L5K7ttysTXvF3aQOegxkOfFhFvqPYJm/I0J5e2U4yRbwaaTyG2acfdIFLu/wwJUEjN7iQ9pj+uCRqfPEtZj1xe6CxXYBQk932gq2H+Y9QsC898Ubq1G9js+AaT3vSIUenGzhGYZl4VIVWjN6h5JvyxAUAbhBSD3zvV1bx/vv1k4e9l7vk3HyZncEZyL81gIB49VKUkWtef9Z/+5778xFjYrz5ykqdkCZEPa6IpphQz9P59MB64rqfK6VNLHN0LwiGxaLP+ctnx6FLXxVYgUtGQS4bbzPhYsqht1FXx3DIecGye4M2l2MorWvaycmwIp0B+OV6sOdcUc/o13ahFour3764lK7C/fYFe2pwRe5UleJEOd+j8lI8ImLgwM9Aclie4v3mjAOAmaxW1m668X9dOtuyQ== root@nixos diff --git a/cluster/compute/pubkeys/c0-13.pub b/cluster/compute/pubkeys/c0-13.pub new file mode 100644 index 0000000..8ba0eb6 --- /dev/null +++ b/cluster/compute/pubkeys/c0-13.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4wc7cxQGddBOqEbYRIo61cOsil//rkK2OKcuBEVXm4MlkYc7ZZIdSs34YAloPnuioTs1csnsGCGkUA1z4zLHIs/o/wLn1pTmA3ZYVevdaFl6CCDBgO/Tv/KXr/g3hq3Dhv9+2mzbu0Zc+44GBH+NrP77DsJAQiEkDH31U8MzqjMb0pzDbF2qI6GBa0IQPORevbJm3MT61MXvxBResYFDZhmgxA6HUokiUsUm0JImgDctwKAYZyzB9p19rZ9aPJxtnvQzh0EuBQc0Cfi8mlJ08DHFVK1F8/iQ98j0eOkTQuA+cqUBc2r0S5Ea0rK++qfEj0KsZy1av9nYlirUg9xNaWd9m/gMf9uMqY2eK3ly3+eSap8u91XIqm9NjFnLhjw7MXUmA+rXzU5gyfpcBu5pZwRQIMs7y5P42sll/kOhg9/VOmdnQqecvUm2ro10fQS8nRthLwVwHNsPFr5JSe36aKbF4bQIYZCBPmz8nLs5THdX3ONREfBRk3T0m5Tm19RvxWSyuaxJlvpTguZ25xk5J+1Qw6jocSsg75u+lP0X/Ty9Q7yCoEbNnLiQXpIU3IAl7RcLk9Edzj4xcpt5QLn6yOdwjn0qrYzke5lClX0l9btixYCyT6r6Z6NvKwJQBJHSDG/I74Brwt5Tr1SRpI6EvywF3rKKL3m9eKVGlrrNt6Q== root@nixos diff --git a/cluster/compute/pubkeys/c0-14.pub b/cluster/compute/pubkeys/c0-14.pub new file mode 100644 index 0000000..10f0c64 --- /dev/null +++ b/cluster/compute/pubkeys/c0-14.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVoWZnPzLuLfiTH82RIAweOqGfdZvKJe2LkaD39rPFr8w5gEIIYmorquYndpSynMH9P4N/HZZ0/kvyjFps4i6uZrJ6gBIZkFCayN2ljyf4/7kvIGlSQQ6c7a1Bty9oFIZYbsHFNNbCUqu8IF1aT3PBoiAjLZFkQlAaDUanIHb34XgDvaweU+s88QG0q7f7MN8eUlBkY4l34pDS2P28xfhbUl5xkDU5/cQXwZxdETsLgwPweXuCDQ58ZqYwqHUrp7GNfJ5QF9CnqTClrRKxfxJeUcDkvBnTWAvIAzMheWouBypyEguuaZDmvp80ylNKxkin18LFC61wOVHK6/1tDRp45CJ3Vhb9T+ch0ys0k3Qg/ebyCkKx1mOuV6waMq7imjWN4sQh7p6BjbrHuxFilczfv87F8p7TlzBMGDh8+ffL7G0ude578kEf05dHZT1M5o859l7iANgPYgEeaAocnZhwWy/0AYSG/zZeEVNCRS7YO0mTOwzPNmf1DhxArX9DrVPODwc7iHRVEy4KnqW+XcP8TcLQXOlqGEIIa+iE/AJlm2siqreoaUNsl9Gh48aiTX/90uzDnnIsMkNRDD05h0JwI1I7itAcTJVs6BX68CyIhD0VbNILsFqTGOHnJWiBR8z+79XnZwheZPM7xhv0n1mTvmi3syB8kZq6sGL6q7zTmw== root@nixos diff --git a/cluster/compute/pubkeys/c0-15.pub b/cluster/compute/pubkeys/c0-15.pub new file mode 100644 index 0000000..cef05ba --- /dev/null +++ b/cluster/compute/pubkeys/c0-15.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEniX5RkR0XcPBj+hZd0sC/1RdQhD/P6rvET55XWmGfkfnfhkMi5vHCv976dTk+H/oee1l/J2cfl6VKQpWGneiiH4ZRD6nqM7Du9AChWYFqP05x4vMag6RYO84qWbIKOE0sJYe52c2Q1TA3bZhaZZIojww/z4En1QDGsM3Gt0NqbyiLKlp5C3AVB+mF47vVb2eR32cnYmwa1Ab0ylQeLnVDtMg2QdXBpAQk5HbwXpyfzmH7VJnKVmF1cGei8LaeXB0M/EbbZMQJCXjrBpPQS3JfMPyFYfd7ok3vVvQiukkzRX+1Y7rWbDw20DHC7cQxFQLUU0cwSU2H6Kl+jFbMA40aQw9cZqUIet500p7SGK/32bZJxcV25XLWFBTAVD96mJhK3WPzeubI8tPmFI0Me7H9AMFgO51vcxqqVFzynF393k+DxGsMewWPJyrnar609j2CLRYjLiEDOjRD8Gywwo5QtIxmCMlaJhAzaJVr125Cp4JIxzKFqPfDsi9SmLfdLMlczQjQAvF0kS55laMXBxgM7ZHX1ihKq6bOnx4Lc1cKTeiLVH84srkhmHgY/G+m/eryQaY7feEVmqwbQovwEnCb+trhttibnOIilbTTVH71qE1rZL+hzgeUwL7jheCPzUQAWzdPvLLDYJq0yjGugHljRgtq7q1zFqrLGIaEgGjcw== root@nixos diff --git a/cluster/compute/pubkeys/c0-16.pub b/cluster/compute/pubkeys/c0-16.pub new file mode 100644 index 0000000..228f083 --- /dev/null +++ b/cluster/compute/pubkeys/c0-16.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClq1v4N8cIxay8hXNOwU4ulrnf7Yyvnyt5reijIfOUfMiGskgi//kAdf5LyaQjyoZAApABTATVU/vIKhc/D2x96h1VJc1j+i9UmlUuawf7r5d5vcfar8Mc5T9cvXHOs/AWsjlH3vp5Kxac/A1+uk5VCGcML71iBRov3bZaM8RT2s+hzt5hu0RvclLPYuzPafKdsMJQg7dw8Y6pb7R45+8729ItKMHcK54IUxiOjLMbNCkHyUy6m0pPBTfj8G2PnoFkFdg77O4lVQnptyFU9wxSa76RLEqQp1CftaNPzxF6hD3VgGjIKSuiRz30A7J37lWMur6H5185NjWefDUyADa8JY5blb0BAP+csOrGxv39Ghh0wchE9I6Lozb+wGKphk4Esbq0wtKCsph3AZptyO3WCZeajQK5sZCawjOZUY0/t0cvtDefreDg0Db/cw1M6vc1fDgfIrHQ7Yq+y9QQmrClW+1b6pcv5DBKiipa9fUOZNkFcW+do5RKqr52IkuhGcG0pb5mbiGSBIb5EqgoEqBi2o5E5RQ8nWf9tF3xCrksCOjAnu5uUUoxPi14NC7z7p6fmwD3xvA/K3BZYkz1SrsccDcpNCSagPvxzdNTfBiW0USIMmWLQQ0lW7j8GjMGK6B0usNApj1WkiSHuzHNMQLVARaZGANXi/T8JfQoglglzw== root@nixos diff --git a/nixops/ekman/pubkeys/c0-2.pub b/cluster/compute/pubkeys/c0-2.pub similarity index 100% rename from nixops/ekman/pubkeys/c0-2.pub rename to cluster/compute/pubkeys/c0-2.pub diff --git a/cluster/compute/pubkeys/c0-3.pub b/cluster/compute/pubkeys/c0-3.pub new file mode 100644 index 0000000..2b8a282 --- /dev/null +++ b/cluster/compute/pubkeys/c0-3.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDmF23dxQHzmN1rVlDkHelCBK48NFc646LFZyB5Gn8cphpur9wV26iOxWOXct+ITU9p6fWmXDko9zCrCWcsF6Ib3rBzbCHKMzF6Vuj0UZg3XP4IUrOdw+Cd5L8GTVoAzskXL0EM+upKPeXFBob8OS3F9Ax0X9ZT66vqY5QGuZWtUikWnakTxbl7JFj+Hgx5/ZzS5cH2e/9NoKkl7wGSkUp0rqjfIEimXI6SNl98kJUtCWEWyUvon4yLuJ4X3+4wrjwdgznspOWgCt0XnTnmak5ADrbz+VuXnDR8TB4AJNfhFySW6E4ZRSrGpZrt1FfaUBxTCmq2Zq/t1chA0Z6z+/k2vCjd9CXodAUalgVmJZ7EQZdSSo8nXA4P5Y8I6TMesTKlBVxMlxqsQfwve+xaPV0/+QSXCvpdRep3+QDVZXaOO4N7UU2xeQR38cNHNzovMy1z5439nq4p5uoA3CxWLL/z6vsZQ3s3xXdvlvPudGWmTW2c0FGKS8ALSWUKLibvPHsEEdIwLJ6FtUs0Vp3QtBKdq/NmN05gZJEOrnfKr8JpvVl9vzRAc8wCYsTyujHTkkYb/E+fb7VsEngbD8iu6PiRgeOui/t0QkTl1ij+SD0YnDXrh61YhQjuvdogJq47gtcSbCE8Q2rkzDpDIEOJNZP7uZINyl9VIRhBb6jt7CC/kw== root@nixos diff --git a/cluster/compute/pubkeys/c0-4.pub b/cluster/compute/pubkeys/c0-4.pub new file mode 100644 index 0000000..be54298 --- /dev/null +++ b/cluster/compute/pubkeys/c0-4.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCmlNP7HKPGYtciCLRL6KyPs1JPF/ehYWuMmeiJrVxdq3YSrfAddB/6cdHMOw/RLukdyLdIwvRfGhy6QZq9BbKBFSJKypBGNe3KTTT7zd/qxNKEbQhJ8m43NtBbOUHBPQ49zE2eunGwMYKL2EW/7ndbtzQOTTrpuZTsGXB6W6Rn/ZnYZC9wXmOyn5XJKleOhAlor0wIyORdAPwdY0s9LdJ6lJUCUqxstFecuqIsgnaMUBbzyTl09e2RbtMduEjRH/B7CHBxj8P4oVlBGTVStOAeeqMudrJeXvRssMB1BX9Tu92CVYnr2FsUazJ+s1R0dHqdXFuWuR2oddw3y1Et0RyAoNmgiku+ZwIbqousNZx8UxrGlMII1fi8RdNqmMZE+8LhC3yhztYQloZ8Ak8J8SpQE7av1gvlHKCMD5bxtBD4mNfj3SSdH9TnF8gu1x6zT0A4LdL+PqK6E+2zZlc54DfLT3SwS214kFUtgM4t9VhSC2E+EHGCgZ91U6xmsd4zVGMaGvzRXcyH0rROzCKsP2M6rVssOER1ePXEZUIC+62PC9IY9tM8lFDH2AB/JF/xdLZYtO9pdWG6d6DOKSpiP2qlMILIX9/qivHTCySnyP3lBtJ7THF8zsJ5sTUO53/Grd0mSLSqTm3T12PIwABPWwiEXKMD2pIszUhu44KRAF0sJQ== root@nixos diff --git a/cluster/compute/pubkeys/c0-5.pub b/cluster/compute/pubkeys/c0-5.pub new file mode 100644 index 0000000..76387a8 --- /dev/null +++ b/cluster/compute/pubkeys/c0-5.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+rokHymxlz8S5jYU6Vamp1TVX+M/rAHGflMWwpAYND7WSt1LkCICHbgAWx9lCXGDEidMuajENtu62tW2zar3cXRrk/p5SdwCb+NiztQ4uGTzGIMxYA52Cjy6KyMi7pG9C6/91WyHFy9aKPFjbA1c2FPgf7eqIYg9B3uBu75yCWCuZ3ZRs+al3R8Tbqo4J7sSanxWRpdc0t7eU0U/rMeTi41SXmNF82smHaQ7iOnWajHAWwybmNpWQkiFpvyahzct0No606oODFl/SttI8R9CCJpvlEeT/Q+nj0hgMciw3J604EJaaLgaYfUcJg/V4F3fe8eGDiOzBYBYrYguNv/XVUW0llP2K+SG0zi/3luQTMghpd+fsGr49cdwFrJWXJhqvE2O5QlQ3mWw+pwlUY7Qgs2usGbQK04VGpJlFiSkso4pythuNMFB37YZoA57+RozsjpeiVmL37xcq6jfsd7ljvjbUICCGLaaJN7p1c3OQSw1pma1R3G14FvefhUQe+F2DLyLfvFq476KN0TSUU42tw9iKfCbP3geZp7UkqpyONerViP9VOa4CQclyuaZ5Yg1x4v7zmfQ6J8J1+sqSwEUe7UnaPuKM2XklViT0AUy33EKIkzGe91aBOwcQMaA7fYHYf1Jvf5lYI5MiA56hkV8f9KVZsKGD9advK+WnsdyUCQ== root@nixos diff --git a/cluster/compute/pubkeys/c0-6.pub b/cluster/compute/pubkeys/c0-6.pub new file mode 100644 index 0000000..62a162b --- /dev/null +++ b/cluster/compute/pubkeys/c0-6.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVFD6R1Mwz0+SqojCUcwLlOm6fKarAPEOB1Ysjo8daGmwx/zfzeHLkzl2RpA8NpSAHheBQJxt+x108wZRZjy+EWYzc/sOaui7lFvZVxxB2ewv1yAoBXUl82O1D2i2ZaaAoLa4G6dZnSfdd0THJNqMIoCCThDaNI3jGcTOzHivVb1XWpdM1DIQtbTcwOJ3Q0UZTiNi5eL+7wlpIKE6/IzkSyTgyYi463CEeR8YXCaILdRweJf5lFem+ahD6uUBe8yuNktpnE8wzQXSHBlNrI3C4Kk9uQicDr1uThexBX8/37hg1V7Wo/AxBQcWlumUCgtnlC53V63XvcRjfB9zrQSbHgZcDnlgGgvMPYillLNCGcFdny1ZbpWpQwlt6kZgHmB3v5m+i63iaCYaAzAoGfp/M/f3mrrQF9GYdsFtngZsGsTxv1Rr9Nmef05V3CewKPaLw13XwPbH+Muvzdx00P9W7P2VmryQXYmAVs4iczUIdRGaTDGJIou4qycRHMAfC6aYEvR2s0+oTzLCMdYgjkVDHxJDkrgSP0Z8++bJa2qPASrTDnT7Hpghup7l1NspJpiZX3vIBLOi+0cBfoRZuLrQ45dDCrRdCySDvv5PfngACqNWt8PCHBeQAHS81d7+xb0e92VfxtZEABTwHlWePMuRmSiSXBi/UGdruFsFm0n0qPQ== root@nixos diff --git a/cluster/compute/pubkeys/c0-7.pub b/cluster/compute/pubkeys/c0-7.pub new file mode 100644 index 0000000..903cde8 --- /dev/null +++ b/cluster/compute/pubkeys/c0-7.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4dNgyC5n1PQ9hp+o1HuTVt7MXkt5Y7wOFAmR+dzWWQ5X/3p7umJzl/I0955CuaXPTb6E6Oqi8WQAlqlBhk9w8qoqoczXsHbqSbnoWSk82VgvHxPv2+5iWvth/FvuTEiM43tt2AhDxG/8TobR2eFD2/2h/T7OpJUcZ7aHT6BZnN5IS/P6MgarOUootDNx9zn3BXFM4Pg8lnB9dnKm4iO7Ju9kk9s2eWzvk37QpNE+H/h58DOMExUcrDzftXNrVbUeZjw7Ho9MER+fdbucrcdKxFH/NiwXlgRSRd+Z5Y+IG+WJdkdJw1hW47MsGRGH+yEq/woE4YvzdcqqqLh9m2a2ucQkk1pe3xpYXCxwS8aVcIDzlja7rGaa2nSBz+Lkp+G5awT2Sg4ob7vQYIN2X6p5zYsqLfWdU5GloWgjEB/zvADjF7qKeEdFFv/eXzaiwnXSoOofcCZTW0qAldK7cdO/esDUfQGIsti8qk1adXUGXG52UhV0p+rMtloW2AAJVFsUiUWvp2Ue3i+p+nAC17aDMOq1UrR60/n1FcjM7mbwAjLgwQ40tLjc54HoMVg92lMTVgJgk+gBYKFsfrI3nbSmvUiRFZDWzPFSno9WtFasWrNCiQ9srmfP0Ud6ro0QCtnvSxFMuXiVufaUlRka0SLlZbA8N3mLrZ0M+7rWy5syLSw== root@nixos diff --git a/cluster/compute/pubkeys/c0-8.pub b/cluster/compute/pubkeys/c0-8.pub new file mode 100644 index 0000000..9c86b14 --- /dev/null +++ b/cluster/compute/pubkeys/c0-8.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBcImgez4RkX1TuqDTk/AUugMqSfsIjdWmEyJRgGw0ZqYSiz9R0yHVM9E6tFIXkGw73RsF1qVQorcw3ZRrxiYLtJHRyBnX0/6IHFwT5w1TezH7LMJmId3ys+LEcpqnCdVaPo2RVsq3RJhnM4dImVC36UUJj8TiOzhZ03lTP7TCPABxvwzn//+T5EjyKuK7acFU4ts4ii7B28dWErCVjsq29uWxKXopfER+45DjhiilQgJ5R15voBNvvFCR4bfnVrsyBwHXXRohAOU1yJYpsUDkXMDY2DIG41LqpP8lg8IyPF36Vwt4w8vmK6rpxBvy4SKg4TIoxUK3eqODnrgN15/2yDTRWWF8u6rpEopWRQEUlKqZvHnXFWuSN5TGUBeOrjqom92udfGrW0l6Yn6iUEt2n+OkPxRuDAGGwjRf4oV2RSqbhbIKfctJg0b+8VhrecNow2zyNlRvfLiQvKotQEDolvfnOYyIWujsjUWttl5a0eoJBm40QgNvWHJWfuaheMhVfEWcfoqgkvZDfBeE11Z/1eogCXMWELJaX/gevE7I+0veu8myzmRwXQOEbk+Ya8tVCSm6YrZjp7flUDRNMW9bZswEdS/ZjvIlK5RgDCAakBoU4MY/PZipYujFoT4AQTNiV3AfuakktXY8dORUHDOcD8zmdbE6B/6t7d1SrHIOow== root@nixos diff --git a/cluster/compute/pubkeys/c0-9.pub b/cluster/compute/pubkeys/c0-9.pub new file mode 100644 index 0000000..dc6705c --- /dev/null +++ b/cluster/compute/pubkeys/c0-9.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSuaHY3hT//NqibrlVTpe4rBOixEVENAS4XG7cfM0pGZlVeRH1ebTki849pdpQ93j+r4Am4kTLYpIq5RHL+Pzpvp7FlY6DgeS/OCtyXNIC3gGe9xJU2khxRWkbYy7PXnjcFY19djy9Tkdgd9oIJqGYc190t9PPnwhG2TGo6LT6t0kTh0LEC3culgU9EAFAHkPaqnGjs5ElRknOqZ1M3jzLMKuL3+ScKRk33q+jb0+Gw/4Gl9fv/BPWQclP7E4PQaddc9oe/SqMN2lXMVV93n+QW/bImBtCAT6873h6wjDlKUmXu2WZPafiDjavbKLpgMuUSXzq9COxHGYN7cbqJENskzxz0awvRE5cBOL85qfZwkdN2uaKbdjRWp+EEcAWdLxWX/QlUS9PruQL1bVNe956nzyqsF3BH1f0M/nLuOv3ArJ0qGS1HgjJT5doVAUl76TqyNy37WZdIDVBFtsmz9XKZfei/1QzY0CKAFw8ZLKb+w68b19A8zrFNwH1W+lWZwxFGmlmn7kj0aoWYitQiDQ74eu0EJDtHgJHvYIgm4gmkXVLwMPPmg+GYJ1pVvh3ychWsZrHOoIEigZP8t/6vAZSFI7DWbMTXfn/aYSuM1qHhe1L2SqqjfdPwbf0EZjlUmj7C5JPv3a3QsGmUPdeugOLs3ow14iWH9k/NRTYzEUmDQ== root@nixos diff --git a/cluster/etcdCluster.nix b/cluster/etcdCluster.nix new file mode 100644 index 0000000..44f6af6 --- /dev/null +++ b/cluster/etcdCluster.nix @@ -0,0 +1,12 @@ +{ + enable = true; + existing = true; + nodes = + { + ekman = "10.255.241.100"; + nfs0 = "10.255.241.80"; + nfs1 = "10.255.241.90"; + }; +} + + diff --git a/cluster/hive.nix b/cluster/hive.nix new file mode 100644 index 0000000..01ebf14 --- /dev/null +++ b/cluster/hive.nix @@ -0,0 +1,22 @@ +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; + # }) {}; + pkgs = import {}; + + ekman = { + deployment = { + tags = [ "ekman" "frontend" ]; + allowLocalDeployment = true; + targetHost = null; + }; + imports = [ ../configuration.nix ]; + }; + + compute = import ./compute { inherit pkgs; }; + nfs0 = import ./nfs0 { inherit pkgs; }; + nfs1 = import ./nfs1 { inherit pkgs; }; +in + { inherit ekman; } // compute // nfs0 // nfs1 diff --git a/nixops/ekman/hosts.nix b/cluster/hosts.nix similarity index 81% rename from nixops/ekman/hosts.nix rename to cluster/hosts.nix index f5f6f46..6522a6a 100644 --- a/nixops/ekman/hosts.nix +++ b/cluster/hosts.nix @@ -1,8 +1,11 @@ '' - 10.255.241.80 nfs0 nfs0.cluster.local - 10.255.241.90 beegfs0 beegfs0.cluster.local + 10.255.241.80 nfs0 nfs0.cluster.local + 10.255.241.90 nfs1 nfs1.cluster.local 10.255.241.100 ekman ekman.cluster.local 10.255.241.100 etcd0 etcd0.cluster.local + 10.255.241.80 etcd1 etcd1.cluster.local + 10.255.241.90 etcd2 etcd2.cluster.local + 10.255.241.101 c0-1 c0-1.cluster.local 10.255.241.102 c0-2 c0-2.cluster.local 10.255.241.103 c0-3 c0-3.cluster.local @@ -20,10 +23,12 @@ 10.255.241.115 c0-15 c0-15.cluster.local 10.255.241.116 c0-16 c0-16.cluster.local - 10.255.243.80 ibnfs0 ibnfs0.cluster.local - 10.255.243.90 ibbeegfs0 ibbeegfs0.cluster.local + 10.255.243.80 ibnfs0 ibnfs0.cluster.local + 10.255.243.90 ibnfs1 ibnfs1.cluster.local 10.255.243.100 ibekman ibekman.cluster.local 10.255.243.100 ibetcd0 ibetcd0.cluster.local + 10.255.243.80 ibetcd1 ibetcd1.cluster.local + 10.255.243.90 ibetcd2 ibetcd2.cluster.local 10.255.243.101 ib0-1 ib0-1.cluster.local 10.255.243.102 ib0-2 ib0-2.cluster.local 10.255.243.103 ib0-3 ib0-3.cluster.local diff --git a/nixops/nfs0/default.nix b/cluster/nfs0/default.nix similarity index 59% rename from nixops/nfs0/default.nix rename to cluster/nfs0/default.nix index e82f25b..4834e41 100644 --- a/nixops/nfs0/default.nix +++ b/cluster/nfs0/default.nix @@ -1,15 +1,17 @@ +{ pkgs ? import {} }: let # Pin the deployment package-set to a specific version of nixpkgs # pkgs = import (builtins.fetchTarball { # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; # }) {}; - pkgs = import {}; + # pkgs = import {}; + etcdCluster = import ../etcdCluster.nix; name = "nfs0"; address = "10.255.241.80"; in { nfs0 = { config, pkgs, ... }: with pkgs; { - # deployment.tags = [ "fs" ]; + deployment.tags = [ "fs" "fs-data" ]; deployment.targetHost = address; system.autoUpgrade.enable = lib.mkForce false; @@ -31,9 +33,16 @@ in { }; }; - cluster = { - k8sNode = true; - }; + services.udev.extraRules = '' + KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + ]; + + cluster.k8sNode = true; features = { host = { @@ -50,22 +59,36 @@ in { ''; }; - certs = { - enable = true; - caBundle = ./ca; - certs = [ - { - name = name; - SANs = [ "${name}.cluster.local" address ]; - owner = "nginx"; - group = "nginx"; - } - ]; + k8s = { + enable = true; + node.enable = true; + master.enable = false; + inherit etcdCluster; }; }; - system.activationScripts = { - kernel.text = '' + # certs = { + # enable = true; + # caBundle = ./ca; + # certs = [ + # { + # name = name; + # SANs = [ "${name}.cluster.local" address ]; + # owner = "nginx"; + # group = "nginx"; + # } + # ]; + # }; + + systemd.services.rc-local = { + description = "rc.local script"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' if [ -e /sys/block/md126 ]; then echo "deadline" > /sys/block/md126/queue/scheduler # echo "4096" > /sys/block/md126/queue/nr_requests @@ -73,8 +96,7 @@ in { echo "always" > /sys/kernel/mm/transparent_hugepage/enabled echo "always" > /sys/kernel/mm/transparent_hugepage/defrag fi - grep -q rdma /proc/fs/nfsd/portlist - [ $? != 0 ] && echo "rdma 20049" > /proc/fs/nfsd/portlist + grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist ''; }; @@ -102,8 +124,16 @@ in { address = address; prefixLength = 24; } ]; + ipv4.routes = [ { + address = "10.255.242.2"; + prefixLength = 32; + via = "10.255.241.100"; + } ]; }; - interfaces.ibp65s0 = { + interfaces."ibp65s0" = { + useDHCP = false; + }; + interfaces."ibp65s0.7666" = { useDHCP = false; ipv4.addresses = [ { address = "10.255.243.80"; @@ -114,6 +144,7 @@ in { allowedTCPPorts = [ 443 9000 9001 ]; allowedUDPPorts = []; extraCommands = '' + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE ''; @@ -122,24 +153,23 @@ in { # services.nginx = { # enable = true; - # statusPage = true; - # virtualHosts = { - # "s3ui.oceanbox.io" = { - # forceSSL = true; - # enableACME = false; - # sslTrustedCertificate = "/var/lib/secrets/ca.pem"; - # sslCertificate = "/var/lib/secrets/s3.pem"; - # sslCertificateKey = "/var/lib/secrets/s3-key.pem"; - # serverAliases = []; - # locations."/" = { - # proxyPass = "http://127.0.0.1:9001"; - # extraConfig = '' - # allow all; - # ''; - # }; - # }; - - # }; + # statusPage = false; + # virtualHosts = { + # "s3ui.oceanbox.io" = { + # forceSSL = true; + # enableACME = false; + # sslTrustedCertificate = "/var/lib/secrets/ca.pem"; + # sslCertificate = "/var/lib/secrets/s3.pem"; + # sslCertificateKey = "/var/lib/secrets/s3-key.pem"; + # serverAliases = []; + # locations."/" = { + # proxyPass = "http://127.0.0.1:9001"; + # extraConfig = '' + # allow all; + # ''; + # }; + # }; + # }; # }; fileSystems = { @@ -147,6 +177,10 @@ in { device = "/data"; options = [ "bind" ]; }; + "/exports/ekman" = { + device = "/data/nfs-provisioner"; + options = [ "bind" ]; + }; "/exports/opt" = { device = "/opt"; options = [ "bind" ]; @@ -157,9 +191,12 @@ in { }; }; + programs.singularity.enable = true; imports = [ - ../ekman/cluster.nix + ../cluster.nix ./hardware-configuration.nix ]; - }; + + }; + } diff --git a/nixops/nfs0/hardware-configuration.nix b/cluster/nfs0/hardware-configuration.nix similarity index 100% rename from nixops/nfs0/hardware-configuration.nix rename to cluster/nfs0/hardware-configuration.nix diff --git a/cluster/nfs0/nfs0.pub b/cluster/nfs0/nfs0.pub new file mode 100644 index 0000000..8fa4abf --- /dev/null +++ b/cluster/nfs0/nfs0.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7wnYBwAdcp5+y7QOKzjd2apRv+Nvquy+qr3jVBeiQgbGsjlpGJBRsGv46QhSbB9QLuFuCUluuvilYbS9/ubm+BtRCrFViAHwFx+QhbLYlAXJPLGfXLxJZDSxbM6c9L+N+GAAfhiCfdhUVB39G33v5dd/VD3J4F/xC7lkPJU59pGokqe9dGrDta33pemSzR9QV9RuV4ZxJG1ld1B2L6TjSkAPTb79PPvMw8LnTViNffmHKp5qvwgIfwRvMApG5KnHw720pPncTW4ejY+utSbP4DcwD2qayKbxnsYDpimMDQxR3id6meZa5LcilxExmz/ig7Fpc3TB3wIpT4ha6VVu2FhRThKR5zD1Qif8BaKy5eAmqcWKQm6M9W56LmUdn/OHpzmaSv47UgtJ+1JG4LqGSaShs8kfVWtaqmev6vvnHBZMm7ni3xyROTSWnjJHLDQJbCYb1PBbct2Dod6MuU54CJUuxw0k+hCJ1QMz1ibme3+LauXVuLJ2JrnAoz7x2CcUv9e5MVx0H4HpWTCxi7cNBO5bbeuxMwHbl3JrZ58LBK9zZYl+nr2F7CZqLyoMimQuVpXT6Aw0k/nfYMAXjUJ3qbBldiGsRgpgbvL27nJK0EhQ8Kd36QhHfdJKhiAIPV+06WVWhPllaqay4OgZIQM1aB83eHg/OUxbOWEu89d7w/w== root@nixos diff --git a/cluster/nfs1/default.nix b/cluster/nfs1/default.nix new file mode 100644 index 0000000..9c48e3c --- /dev/null +++ b/cluster/nfs1/default.nix @@ -0,0 +1,152 @@ +{ pkgs ? import {} }: +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; + # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; + # }) {}; + # pkgs = import {}; + etcdCluster = import ../etcdCluster.nix; + name = "nfs1"; + address = "10.255.241.90"; +in { + nfs1 = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "fs" "fs-work" ]; + deployment.targetHost = address; + system.autoUpgrade.enable = lib.mkForce false; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + services.udev.extraRules = '' + KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + ]; + + cluster.k8sNode = true; + + features = { + host = { + inherit address; + inherit name; + }; + + os = { + externalInterface = "enp33s0f3np3"; + nfs.enable = true; + nfs.exports = '' + /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + k8s = { + enable = true; + node.enable = true; + master.enable = false; + inherit etcdCluster; + }; + }; + + # services.kubernetes.kubelet.extraSANs = mkSANs { + # name = cfg.name; + # address = cfg.address; + # }; + systemd.services.rc-local = { + description = "rc.local script"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' + # if [ -e /sys/block/md126 ]; then + # echo "deadline" > /sys/block/md126/queue/scheduler + # # echo "4096" > /sys/block/md126/queue/nr_requests + # echo "4096" > /sys/block/md126/queue/read_ahead_kb + # echo "always" > /sys/kernel/mm/transparent_hugepage/enabled + # echo "always" > /sys/kernel/mm/transparent_hugepage/defrag + # fi + grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist + ''; + }; + + boot.kernel.sysctl = { + "vm.dirty_background_ratio" = 5; + "vm.dirty_ratio" = 10; + "vm.vfs_cache_pressure" = 50; + "vm.min_free_kbytes" = 262144; + }; + + networking = { + hostName = name; + interfaces.enp33s0f3np3 = { + useDHCP = false; + ipv4.addresses = [ { + address = address; + prefixLength = 24; + } ]; + ipv4.routes = [ { + address = "10.255.242.2"; + prefixLength = 32; + via = "10.255.241.100"; + } ]; + }; + interfaces."ibp65s0" = { + useDHCP = false; + }; + interfaces."ibp65s0.7666" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.90"; + prefixLength = 24; + } ]; + }; + firewall = { + allowedTCPPorts = []; + allowedUDPPorts = []; + extraCommands = '' + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; + }; + + fileSystems = { + "/exports/work" = { + device = "/work"; + options = [ "bind" ]; + }; + "/exports/data" = { + device = "/data"; + options = [ "bind" ]; + }; + "/exports/opt" = { + device = "/opt"; + options = [ "bind" ]; + }; + # "/vol/local-storage/vol1" = { + # device = "/vol/vol1"; + # options = [ "bind" ]; + # }; + }; + + programs.singularity.enable = true; + + imports = [ + ../cluster.nix + ./hardware-configuration.nix + ]; + }; + +} diff --git a/cluster/nfs1/hardware-configuration.nix b/cluster/nfs1/hardware-configuration.nix new file mode 100644 index 0000000..9a7c90e --- /dev/null +++ b/cluster/nfs1/hardware-configuration.nix @@ -0,0 +1,49 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "sd_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/19d00648-5fbd-4464-93a0-e013d7f79d3a"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/68AF-2717"; + fsType = "vfat"; + }; + + fileSystems."/data" = + { device = "/dev/disk/by-uuid/bf1bede5-bc60-4603-874c-88ed85e6ab5f"; + fsType = "ext4"; + }; + + fileSystems."/work" = + { device = "/dev/disk/by-uuid/31552727-950f-4b3a-91fc-5af6b989b1d3"; + fsType = "xfs"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/nfs1/nfs1.pub b/cluster/nfs1/nfs1.pub new file mode 100644 index 0000000..908bab9 --- /dev/null +++ b/cluster/nfs1/nfs1.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/a5Q1Y8hta/v8W/cM45o6UimPLfSzz6NM+WaNAYbwnLoz2VVowHdjyndajmH1CMK/cTDRiVSrTkky48f3JNKzCMjmwVA6PjWmRxF0tqcitW9hzFB+L10+fgBsZu7MUMhuWfTtekIgnRjqNZIA3q5PGz9JrMHsRem+RGmDayaNSAkOZ1ToGSS7pnGm06qAswLJaK15wnupw5x9NYO1RXq2Dy2pX68e4urA+h9X9tqDfb6kQP3/9gb+q6BpiGUwUHW5x0HK/LwNqThKV7ZwzgMHqz4rq5TieGemTw8LKcDjj3BWQTDt4SlZ2gux49C7z1vBipHtxKHIdZH9AXYPZCkjgI/OVVJvxkbgMM+3FSr3j/KvTOdL5ycJvGBsF4dJnRTTpay7OmnksQ62+vwNO2dvDnhAgVntrr9Fb1ahLnTN7dOf6BlkUbz+vtYby0gcjr5sW8b4duGpJThDDkAGLUKkx1z94/bpPQpUb1ajXPizO7jjJCK20zTXWG2uiOdXWHmZ3YVoJ/qSOvtUAX2RR2FgJzHqnDnHtmLiMc9OkZ0eJwo6ltBqGHPNXCyBwHAa8eSH2Af+10vfzJ88J8MGD88DV7SYWbJxuB+KMAnDX04UfrvqLnRADzbFuiG8locorXFE90wEZTz/nFWsM+Q/XPA+M1GaO6fK9SyN45LgApKARQ== root@nfs1 diff --git a/nixops/ekman/overlays.nix b/cluster/overlays.nix similarity index 100% rename from nixops/ekman/overlays.nix rename to cluster/overlays.nix diff --git a/nixops/ekman/users.nix b/cluster/users.nix similarity index 100% rename from nixops/ekman/users.nix rename to cluster/users.nix diff --git a/configuration.nix b/configuration.nix index 14926ac..ecd1e6e 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,11 +1,10 @@ { pkgs, ...}: let - nodes = import ./nixops/ekman/nodes.nix; - myvnc = import ./nixops/ekman/myvnc.nix { inherit pkgs; }; + computeNodes = import ./cluster/compute/nodes.nix; + etcdCluster = import ./cluster/etcdCluster.nix; + myvnc = import ./cluster/compute/myvnc.nix { inherit pkgs; }; in { - # deployment.tags = [ "frontend" ]; - systemd.targets = { sleep.enable = false; suspend.enable = false; @@ -40,13 +39,15 @@ in k8s = { master.enable = true; - inherit nodes; + node.enable = true; + nodes = computeNodes; + inherit etcdCluster; }; monitoring = { server = { enable = false; - scrapeHosts = [ "frontend" "mds1-0" ] ++ (builtins.map (x: x.name) nodes); + scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); defaultAlertReceiver = { email_configs = [ { to = "jonas.juselius@oceanbox.io"; } @@ -76,6 +77,22 @@ in 0xe8ebd3030024a2c6 "frontend" 0xe8ebd3030024981e "c0-1" 0xe8ebd3030024a21a "c0-2" + 0xe8ebd30300249a3a "c0-3" + 0xe8ebd30300248b9e "c0-4" + 0xe8ebd30300248b86 "c0-5" + 0xe8ebd3030024998a "c0-6" + 0xe8ebd30300248b8e "c0-7" + 0xe8ebd3030024999e "c0-8" + 0xe8ebd30300248fca "c0-9" + 0xe8ebd3030024a216 "c0-10" + 0xe8ebd30300248b96 "c0-11" + 0xe8ebd30300248b9a "c0-12" + 0xe8ebd303002495d2 "c0-13" + 0xe8ebd303002495e2 "c0-14" + 0xe8ebd30300248f42 "c0-15" + 0xe8ebd303002495e6 "c0-16" + 0xe8ebd3030024a2a2 "nfs0" + 0xe8ebd3030024a2ae "nfs1" ''; }; slurm-exporter = { @@ -85,6 +102,10 @@ in }; }; + services.udev.extraRules = '' + KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + ''; + networking = { useDHCP = false; hostName = "ekman"; @@ -101,8 +122,16 @@ in address = "10.255.241.100"; prefixLength = 24; } ]; + # ipv4.routes = [ { + # address = "10.1.2.0"; + # prefixLength = 24; + # via = "10.255.241.1"; + # } ]; }; - interfaces.ibp65s0 = { + interfaces."ibp65s0" = { + useDHCP = false; + }; + interfaces."ibp65s0.7666" = { useDHCP = false; ipv4.addresses = [ { address = "10.255.243.100"; @@ -110,10 +139,15 @@ in } ]; }; defaultGateway = "10.255.242.1"; - firewall.extraCommands = '' - iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT - iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE - ''; + firewall = { + allowedTCPPorts = [ 4443 ]; + extraCommands = '' + # iptables -I INPUT -s 10.1.2.0/22 -j ACCEPT + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; }; fileSystems = { @@ -217,6 +251,18 @@ in ]; }; + system.activationScripts = { + home-permissions.text = '' + chmod 755 /home/olean + chmod 755 /home/frankgaa + chmod 755 /home/jonas + chmod 755 /home/stig + chmod 755 /home/bast + chmod 755 /home/simenlk + ''; + }; + + # ssh-rsa is deprecated, but putty/winscp users use it # services.openssh.extraConfig = '' # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 @@ -224,6 +270,6 @@ in environment.systemPackages = [ myvnc ]; - imports = [ ./nixops/ekman/cluster.nix ./hardware-configuration.nix ]; + imports = [ ./cluster/cluster.nix ./hardware-configuration.nix ]; } diff --git a/ekman.pub b/ekman.pub new file mode 100644 index 0000000..8d66cd4 --- /dev/null +++ b/ekman.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSfhEB+DPHCWHmdzf+Xea7N9A/Pd98Nh9ChcQqVcjJNUMNeOta3NIdW5m9sDhWlB6kioMEJa4DbdAfJsKciGnSh9VJeerfnPFgov3WIr5RfoTAfKRE5GKW28c1f4m0WpfZkh5gvS0mAT7jkxwvTcQI9PViteIb4muAlbLPCqDUhFWC9yzSHzSt9MSMjpPY18GlsWyPM2ctT3OUq28wWlcleu7pFEFIpLKj+tqpBDBvS1I0fKagR0eDmCUGGbT8Fth+Jodt66k+QmJ28A58DfiDPECeN1/AzhwOMRYYMLSFmrLRmy/gML1egiZM+zwDKTRQ9lHhFjO8SOfq0F/A4NxFXegmXI1QbbtsG8KrLEDd9a8pZG99YdRa1dE5TA3H5iAnekN7rO0vfnlJfGMQOdI2I9JtqX48k98wUjxmv0EiUJV68j0rQaEwDQFjbEmiN2nEA9sm6+CMKCf3PywvDAijzAzSicA5Pm9MfNQLiXTXZbB7biJlNLfSlSOZrOP7Rd1VZ+bF9lYou88Y3/BZ/S23NdMVr9DEVYu2pUOCEgeXwVW32LHEgDfUslp7ZDUICGgLMQX5Wpz8f+zByRUmE2ovhCjcT71TVn/paIqjjy+2sgDX+BVSULQQAoPS56by1jfMIZjjNqDWE86yY3uWkeobmY5C55s8q2Hlyv+8meq6LQ== root@nixos diff --git a/nixops/ekman/pubkeys/ekman.pub b/nixops/ekman/pubkeys/ekman.pub deleted file mode 100644 index 0a83062..0000000 --- a/nixops/ekman/pubkeys/ekman.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCySwxKuwdqn+jaEi73BPSFslUoLW2St3FKZ1iojQa7H73SZDWQGCyyR9bwmMpjWUCXt/7aZmljF78MF9t/dE/ZJAQfxYv/rt6QCYDWLZBalyNQ2X3R7U2WNlZPF3KwnxnmOJOPCDKnHrbsGaUG7z6oERaCSgngY8haOVsHJIo55gpjBCkAT/WhvFQ4vcaECuJWVg619lebDVTAVNZDdI7I8KzezMOSX+dH1Z/nr90pq54ZunK5RmaMuYo7tb/PeD3T1/cbK8PUyZsx29iih8PtOwoIkeIaszvrk1BeJmFkhfUyuXZp3/a2BlZ01pIIf2RVpimsqCuk4vYw3OB8MQZDihBniYaNjvogsl7VGoVfGiJHb1az3P8NUYDl0kn7MaNJC5Mboaqj9DCDTAht6mTl3qLkDvpMnIAbxxRCNwqSxhfUvOZJRVe3qKWeKfggOeAWFwxy/Ij3EpiD5eiA/PnuZEA0iuJ+EPEbjJdlcUxy9ZjGgn3t0f5VygMV9E6v9fVF+79ocWqyuugwO/4WTWC/jqq5dfujNfsQ3QZ+wfsyuyZsDiCsqdk0GS3sE7ngy4aDs4G4qdkzB0USFaydv7R/Rbvsy2sLwivB+pLboVrnAVtDIlHuaKtMR1De/K9G4vvjEQ89T4Os3bMwkMY15HnEX5mO5vARnnz+VfGcKCbIIQ== root@OBNODE02