diff --git a/clusters/stokes/cluster.nix b/clusters/stokes/cluster.nix new file mode 100644 index 0000000..334fec7 --- /dev/null +++ b/clusters/stokes/cluster.nix @@ -0,0 +1,229 @@ +{ pkgs, lib, config, ... }: +with lib; +let + cfg = config.node; + + configuration = { + features = { + os = { + boot.uefi = true; + adminAuthorizedKeys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no" + ]; + docker.enable = true; + + mailRelay = { + enable = true; + adminEmail = "jonas.juselius@tromso.serit.no"; + mailDomain = "itpartner.no"; + mailGateway = "smtpgw.itpartner.no:465"; + mailAuthUser = "utvikling"; + }; + }; + cachix.enable = false; + + monitoring.nodeExporter.enable = true; + + pki = { ca = ./ca; }; + + hpc = { + enable = true; + slurm = { + client = true; + mungeKey = ./munge.key; + controlMachine = "stokes"; + nodeName = [ + "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" + "stokes Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=64000 TmpDisk=500000 State=UNKNOWN" + ]; + partitionName = [ + "batch Nodes=c0-[1-8] Default=YES MaxTime=INFINITE State=UP" + "frontend Nodes=stokes MaxTime=1:00:00 State=UP" + ]; + }; + beegfs = { + enable = true; + beegfs = { + mds0-0 = { + mgmtdHost = "mds0-0"; + connAuthFile = ""; + client = { + enable = true; + mountPoint = "/work"; + }; + }; + }; + }; + }; + }; + + networking = { + domain = mkDefault "regnekraft.io"; + defaultGateway = mkDefault "10.1.61.1"; + nameservers = mkDefault [ "8.8.8.8" ]; + search = mkDefault [ "local" ]; + extraHosts = import ./hosts.nix; + firewall.extraCommands = '' + iptables -I INPUT -s 10.1.61.0/24 -j ACCEPT + ''; + }; + + environment.variables = {}; + + systemd.services."serial-getty@ttyS0".enable = true; + + boot = { + kernelModules = [ "ib_umad" "ib_ipoib" ]; + kernelParams = [ + "console=ttyS0,115200" + "console=tty0" + ]; + }; + + nix = { + maxJobs = 32; + # binaryCachePublicKeys = [ + # "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig==" + # ]; + }; + }; + + deployment = { + deployment.targetHost = cfg.address; + }; + + i40efix = { + boot = let kernelExtras = pkgs.callPackage ./kernel.nix {}; in { + extraModulePackages = [ kernelExtras.i40e2 ]; + kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ]; + }; + }; + + shosts = { + environment.etc."ssh/shosts.equiv" = { + mode = "0644"; + uid = 0; + gid = 0; + text = '' + 10.1.62.2 + 10.1.61.100 + 10.1.61.101 + 10.1.61.102 + 10.1.61.103 + 10.1.61.104 + 10.1.61.105 + 10.1.61.106 + 10.1.61.107 + 10.1.61.108 + 10.1.63.100 + 10.1.63.101 + 10.1.63.102 + 10.1.63.103 + 10.1.63.104 + 10.1.63.105 + 10.1.63.106 + 10.1.63.107 + 10.1.63.108 + ''; + }; + + programs.ssh.knownHosts = { + stokes = { + hostNames = [ + "stokes" "stokes.hpc.local" "stokes.regnekraft.io" "10.1.61.100" + "10.1.63.100" "10.1.62.2" + ]; + publicKeyFile = ./pubkeys/stokes.pub; + }; + c0-1 = { hostNames = [ "c0-1" "c0-1.hpc.local" "10.1.61.101" "10.1.63.101" ]; publicKeyFile = ./pubkeys/c0-1.pub; }; + c0-2 = { hostNames = [ "c0-2" "c0-2.hpc.local" "10.1.61.102" "10.1.63.102" ]; publicKeyFile = ./pubkeys/c0-2.pub; }; + c0-3 = { hostNames = [ "c0-3" "c0-3.hpc.local" "10.1.61.103" "10.1.63.103" ]; publicKeyFile = ./pubkeys/c0-3.pub; }; + c0-4 = { hostNames = [ "c0-4" "c0-4.hpc.local" "10.1.61.104" "10.1.63.104" ]; publicKeyFile = ./pubkeys/c0-4.pub; }; + c0-5 = { hostNames = [ "c0-5" "c0-5.hpc.local" "10.1.61.105" "10.1.63.105" ]; publicKeyFile = ./pubkeys/c0-5.pub; }; + c0-6 = { hostNames = [ "c0-6" "c0-6.hpc.local" "10.1.61.106" "10.1.63.106" ]; publicKeyFile = ./pubkeys/c0-6.pub; }; + c0-7 = { hostNames = [ "c0-7" "c0-7.hpc.local" "10.1.61.107" "10.1.63.107" ]; publicKeyFile = ./pubkeys/c0-7.pub; }; + c0-8 = { hostNames = [ "c0-8" "c0-8.hpc.local" "10.1.61.108" "10.1.63.108" ]; publicKeyFile = ./pubkeys/c0-8.pub; }; + }; + + environment.systemPackages = [ openssh-shosts ]; + + security.wrappers = { + ssh-keysign = { + source = "${openssh-shosts}/libexec/ssh-keysign"; + owner = "root"; + group = "root"; + permissions = "u+rs,g+rx,o+rx"; + }; + }; + }; + + openssh-shosts = pkgs.openssh.overrideAttrs (attrs: { + buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ]; + }); + + myvnc = + let + myvnc = pkgs.writeScriptBin "myvnc" '' + #!${pkgs.runtimeShell} + + uid=`id -u` + port=$((9000+$uid)) + shell=`getent passwd $(id -un) | awk -F : '{print $NF}'` + vnc=${pkgs.tigervnc}/bin/vncserver + + case $1 in + -p|--port) shift; port=$1 ;; + kill|stop) + display=$($vnc -list | sed -n 's/^\(:[0-9]\+\).*/\1/p'| head -1) + $vnc -kill $display + exit 0 + ;; + esac + ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1 + [ $? = 1 ] && $vnc -rfbport $port + echo "Xvnc server is running on port $port." + exec $shell -i + ''; + + buildCommand = '' + mkdir -p $out/bin + echo $src > $out/bin/myvnc + chmod 755 $out/bin/myvnc + ''; + in { + environment.systemPackages = [ myvnc ]; + }; + +in { + options.node = { + address = mkOption { + type = types.str; + default = null; + }; + + i40efix = mkEnableOption "Apply fix for i40e driver"; + + myvnc = mkEnableOption "Enable myvnc script"; + }; + + config = mkMerge [ + configuration + + deployment + + shosts + + (mkIf cfg.i40efix i40efix) + + (mkIf cfg.myvnc myvnc) + ]; + + imports = [ + ../../modules + ../../nixos + ./users.nix + ]; +} + diff --git a/clusters/stokes/default.nix b/clusters/stokes/default.nix new file mode 100644 index 0000000..48ee48c --- /dev/null +++ b/clusters/stokes/default.nix @@ -0,0 +1,203 @@ +let + stokes = { + node.address = "10.1.62.2"; + node.myvnc = true; + + features = { + os = { + externalInterface = "eno1"; + nfs.enable = true; + nfs.exports = '' + /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + hpc = { + slurm.server = true; + frontend = true; + }; + + monitoring = { + server = { + enable = true; + scrapeHosts = [ + "frontend" "mds0-0" + "c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8" + ]; + defaultAlertReceiver = { + email_configs = [ + { to = "jonas.juselius@tromso.serit.no"; } + ]; + }; + pageAlertReceiver = { + webhook_configs = [ + { + url = "https://prometheus-msteams.k2.itpartner.no/stokes"; + http_config = { + tls_config = { insecure_skip_verify = true; }; + }; + } + ]; + }; + }; + webUI.enable = true; + webUI.acmeEmail = "innovasjon@itpartner.no"; + webUI.allow = [ + "10.1.2.0/24" + "172.19.254.0/24" + ]; + infiniband-exporter = { + enable = true; + nameMap = '' + 0x0c42a10300ddc4bc "frontend" + 0x1c34da0300787798 "mds0-0" + 0x0c42a10300dbe7f4 "c0-1" + 0x0c42a10300dbe7d8 "c0-2" + 0x0c42a10300dbe800 "c0-3" + 0x0c42a10300dbec80 "c0-4" + 0x0c42a10300dbea50 "c0-5" + 0x0c42a10300dbeb2c "c0-6" + 0x0c42a10300dbe7fc "c0-7" + 0x0c42a10300dbe5a0 "c0-8" + ''; + }; + slurm-exporter = { + enable = true; + port = 6080; + }; + }; + }; + + networking = { + interfaces.eno1 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.62.2"; + prefixLength = 24; + } ]; + }; + interfaces.enp175s0f0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.61.100"; + prefixLength = 24; + } ]; + }; + interfaces.ibp59s0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.63.100"; + prefixLength = 24; + } ]; + }; + defaultGateway = "10.1.62.1"; + firewall.extraCommands = '' + iptables -I INPUT -s 10.1.61.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE + ''; + }; + + fileSystems ={ + "/exports/home" = { + device = "/home"; + options = [ "bind" ]; + }; + "/exports/opt" = { + device = "/opt"; + options = [ "bind" ]; + }; + }; + + security.pam.services.sshd.googleAuthenticator.enable = true; + + nix.extraOptions = '' + secret-key-files = /etc/nix/stokes.private + ''; + + services.xserver = { + enable = true; + enableCtrlAltBackspace = true; + layout = "us"; + xkbVariant = "altgr-intl"; + xkbOptions = "eurosign:e"; + displayManager = { + gdm.enable = true; + job.logToFile = true; + }; + desktopManager.xfce.enable = true; + }; + + services.prometheus.alertmanager.configuration.global = { + smtp_smarthost = "smtpgw.itpartner.no:465"; + smtp_auth_username = "utvikling"; + smtp_auth_password = "S0m3rp0m@de#21!"; + smtp_hello = "stokes.regnekraft.io"; + smtp_from = "noreply@stokes.regnekraft.io"; + }; + + imports = [ ./cluster.nix ./hw/frontend.nix ]; + }; + + compute = { + features = { + os.externalInterface = "eno33"; + hpc.compute = true; + }; + + fileSystems = { + "/home" = { + device = "10.1.61.100:/home"; + fsType = "nfs"; + }; + "/opt" = { + device = "10.1.61.100:/opt"; + fsType = "nfs"; + }; + }; + }; + + genComputeNodes = idx: nNodes: + let + nodeList = builtins.genList (x: x + 1) nNodes; + mkCompute = n: + let + ip = "10.1.61.${toString (n + 100)}"; + ipoib = "10.1.63.${toString (n + 100)}"; + name = "c${toString idx}-${toString n}"; + hw = ./hw + "/${name}.nix"; + in { + "${name}" = { + node = { + address = ip; + i40efix = true; + }; + networking = { + interfaces.eno33 = { + useDHCP = false; + ipv4.addresses = [ { + address = ip; + prefixLength = 24; + } ]; + ipv4.routes = [ { + address = "10.1.62.2"; + prefixLength = 32; + via = "10.1.61.100"; + } ]; + + }; + interfaces.ibp65s0 = { + useDHCP = false; + ipv4.addresses = [ { + address = ipoib; + prefixLength = 24; + } ]; + }; + }; + imports = [ ./cluster.nix hw ]; + } // compute; + }; + in + builtins.foldl' (a: n: a // mkCompute n) {} nodeList; +in + { inherit stokes; } // genComputeNodes 0 8 + diff --git a/clusters/stokes/hosts.nix b/clusters/stokes/hosts.nix new file mode 100644 index 0000000..260ccba --- /dev/null +++ b/clusters/stokes/hosts.nix @@ -0,0 +1,26 @@ +'' + 10.1.62.2 stokes stokes.regnekraft.io + + 10.1.61.100 frontend frontend.hpc.local + 10.1.61.101 c0-1 c0-1.hpc.local + 10.1.61.102 c0-2 c0-2.hpc.local + 10.1.61.103 c0-3 c0-3.hpc.local + 10.1.61.104 c0-4 c0-4.hpc.local + 10.1.61.105 c0-5 c0-5.hpc.local + 10.1.61.106 c0-6 c0-6.hpc.local + 10.1.61.107 c0-7 c0-7.hpc.local + 10.1.61.108 c0-8 c0-8.hpc.local + + 10.1.61.80 mds0-0 mds0-0.hpc.local + + 10.1.63.101 ib0-1 ib0-1.hpc.local + 10.1.63.102 ib0-2 ib0-2.hpc.local + 10.1.63.103 ib0-3 ib0-3.hpc.local + 10.1.63.104 ib0-4 ib0-4.hpc.local + 10.1.63.105 ib0-5 ib0-5.hpc.local + 10.1.63.106 ib0-6 ib0-6.hpc.local + 10.1.63.107 ib0-7 ib0-7.hpc.local + 10.1.63.108 ib0-8 ib0-8.hpc.local + + 10.1.63.80 ibmds0-0 ibmds0-0.hpc.local +'' diff --git a/clusters/stokes/hw/c0-1.nix b/clusters/stokes/hw/c0-1.nix new file mode 100644 index 0000000..0179b2c --- /dev/null +++ b/clusters/stokes/hw/c0-1.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/3af91585-8079-420d-acdf-f60b94d3cfff"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/3590-199A"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-2.nix b/clusters/stokes/hw/c0-2.nix new file mode 100644 index 0000000..1f638eb --- /dev/null +++ b/clusters/stokes/hw/c0-2.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/d89e1496-fda1-4de0-b2cc-474967b04402"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/A51A-1F4D"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-3.nix b/clusters/stokes/hw/c0-3.nix new file mode 100644 index 0000000..c98b526 --- /dev/null +++ b/clusters/stokes/hw/c0-3.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/84cc9cea-08eb-4b54-8ca3-2aa5c1300a92"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/882D-A342"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-4.nix b/clusters/stokes/hw/c0-4.nix new file mode 100644 index 0000000..c34903f --- /dev/null +++ b/clusters/stokes/hw/c0-4.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f0826ad5-8a4e-427d-98d3-5afa44440993"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/21A6-D34C"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-5.nix b/clusters/stokes/hw/c0-5.nix new file mode 100644 index 0000000..43ad87f --- /dev/null +++ b/clusters/stokes/hw/c0-5.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/37944fce-07ca-492f-906a-620a37e7e1b3"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/2F51-EC20"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-6.nix b/clusters/stokes/hw/c0-6.nix new file mode 100644 index 0000000..ada6f39 --- /dev/null +++ b/clusters/stokes/hw/c0-6.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/d23386a8-0ca8-4871-b662-decf2b24f4d7"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/AF8A-DEFE"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-7.nix b/clusters/stokes/hw/c0-7.nix new file mode 100644 index 0000000..54107e3 --- /dev/null +++ b/clusters/stokes/hw/c0-7.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/12477966-c6c5-47c6-afdc-35fa7e57e837"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/DB96-7453"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/c0-8.nix b/clusters/stokes/hw/c0-8.nix new file mode 100644 index 0000000..d4dedcc --- /dev/null +++ b/clusters/stokes/hw/c0-8.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/2f468316-5832-4684-866d-2e92b08fb68b"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/76FE-F657"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/clusters/stokes/hw/frontend.nix b/clusters/stokes/hw/frontend.nix new file mode 100644 index 0000000..84acc26 --- /dev/null +++ b/clusters/stokes/hw/frontend.nix @@ -0,0 +1,27 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/099be829-726d-44c9-b113-4c2604a663c5"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/8111-9F24"; + fsType = "vfat"; + }; + + swapDevices = [ ]; +} diff --git a/clusters/stokes/kernel.nix b/clusters/stokes/kernel.nix new file mode 100644 index 0000000..ebc515f --- /dev/null +++ b/clusters/stokes/kernel.nix @@ -0,0 +1,46 @@ +{pkgs, stdenv, fetchurl, config, kernel ? pkgs.linux, ...}: +let + i40e = + stdenv.mkDerivation rec { + name = "i40e-${version}-${kernel.version}"; + version = "2.13.10"; + + src = pkgs.fetchFromGitHub { + owner = "dmarion"; + repo = "i40e"; + rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd"; + sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr"; + }; + + hardeningDisable = [ "pic" ]; + + configurePhase = '' + cd src + kernel_version=${kernel.modDirVersion} + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk + export makeFlags="BUILD_KERNEL=$kernel_version" + ''; + + installPhase = '' + install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko" + ''; + + dontStrip = true; + + enableParallelBuilding = true; + + meta = { + description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)"; + homepage = https://github.com/dmarion/i40e; + license = stdenv.lib.licenses.gpl2; + }; + }; +in +{ + i40e2 = i40e; + overlay = self: super: { + linuxPackages = super.linuxPackages // { inherit i40e; }; + }; +} + diff --git a/clusters/stokes/munge.key b/clusters/stokes/munge.key new file mode 100644 index 0000000..0aeca32 --- /dev/null +++ b/clusters/stokes/munge.key @@ -0,0 +1,2 @@ +/ik/|RER$Qfj5rdС7{99TۛiČ,Ќh8tv:%T +|ȴΕV00w|Ϯ|_Y{3L_!F1Td&F7BR \ No newline at end of file diff --git a/clusters/stokes/pubkeys/c0-1.pub b/clusters/stokes/pubkeys/c0-1.pub new file mode 100644 index 0000000..1351c52 --- /dev/null +++ b/clusters/stokes/pubkeys/c0-1.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwwu4RJJhKo8s2Mtmpdvxs02d4IwrmS9cTpfAOQ2YkI root@nixos diff --git a/clusters/stokes/pubkeys/c0-2.pub b/clusters/stokes/pubkeys/c0-2.pub new file mode 100644 index 0000000..c09679b --- /dev/null +++ b/clusters/stokes/pubkeys/c0-2.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBEV8wEkeGUOs6umhdeOKYnVlYlta2rOCZSoezvu+bZ4 root@nixos diff --git a/clusters/stokes/pubkeys/c0-3.pub b/clusters/stokes/pubkeys/c0-3.pub new file mode 100644 index 0000000..0c58f4b --- /dev/null +++ b/clusters/stokes/pubkeys/c0-3.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQFQqo2vcj2eYaH7nwdEzgCPme+7g3Db+s16KbQHzLI root@nixos diff --git a/clusters/stokes/pubkeys/c0-4.pub b/clusters/stokes/pubkeys/c0-4.pub new file mode 100644 index 0000000..df6aaee --- /dev/null +++ b/clusters/stokes/pubkeys/c0-4.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDIsqJqWWftoiyiOOHnIuHYqbsaOg4AbKNm80wpjH2E8 root@nixos diff --git a/clusters/stokes/pubkeys/c0-5.pub b/clusters/stokes/pubkeys/c0-5.pub new file mode 100644 index 0000000..c746bf4 --- /dev/null +++ b/clusters/stokes/pubkeys/c0-5.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDme/xztp22wQOybtN2TVXMcn2QcVaXtRMp4AnPnzr2T root@nixos diff --git a/clusters/stokes/pubkeys/c0-6.pub b/clusters/stokes/pubkeys/c0-6.pub new file mode 100644 index 0000000..b64a074 --- /dev/null +++ b/clusters/stokes/pubkeys/c0-6.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImItUKMSygMY/1ZNsyGkyfywyngEDgZ7TxM63UwG1VH root@nixos diff --git a/clusters/stokes/pubkeys/c0-7.pub b/clusters/stokes/pubkeys/c0-7.pub new file mode 100644 index 0000000..1111820 --- /dev/null +++ b/clusters/stokes/pubkeys/c0-7.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID1WxWnNg+L2+lQ3W2mD0/6kqwuCUOEJImvWnQYLzdUB root@nixos diff --git a/clusters/stokes/pubkeys/c0-8.pub b/clusters/stokes/pubkeys/c0-8.pub new file mode 100644 index 0000000..5c92679 --- /dev/null +++ b/clusters/stokes/pubkeys/c0-8.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBq+v2bBhQieqfXmtmGYb/9mOv9oc88zerRkkpGpoc0x root@nixos diff --git a/clusters/stokes/pubkeys/stokes.pub b/clusters/stokes/pubkeys/stokes.pub new file mode 100644 index 0000000..b4c0d9c --- /dev/null +++ b/clusters/stokes/pubkeys/stokes.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkSSuHkieXwgFMKRy4MjwjNrJEItWbQHeAAH+Zn1YZp root@localhost diff --git a/clusters/stokes/users.nix b/clusters/stokes/users.nix new file mode 100644 index 0000000..678c619 --- /dev/null +++ b/clusters/stokes/users.nix @@ -0,0 +1,224 @@ +{ pkgs, ... }: +{ + + users.groups = { + jonas = { gid = 1000; }; + olean = { gid = 1001; }; + frankgaa = { gid = 1002; }; + hdj002 = { gid = 1003; }; + hes = { gid = 1004; }; + mad = { gid = 1005; }; + peyghamg = { gid = 1006; }; + qin = { gid = 1007; }; + eli = { gid = 1008; }; + ovanov = { gid = 1009; }; + bast = { gid = 1010; }; + # @grp@ + + sif = { + gid = 11000; + members = [ + "jonas" + "olean" + "bast" + "frankgaa" + "hes" + "hdj002" + "mad" + "eli" + "ovanov" + "peyghamg" + "qin" + ]; + }; + }; + + users.users = { + jonas = { + description = "Jonas Juselius"; + home = "/home/jonas"; + group = "jonas"; + extraGroups = [ + "users" + "wheel" + "root" + "adm" + "cdrom" + "docker" + "fuse" + "wireshark" + "libvirtd" + "networkmanager" + "tty" + "keys" + ]; + uid = 1000; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" + ]; + }; + + olean = { + description = "Ole Anders Nøst"; + home = "/home/olean"; + group = "olean"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1001; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAlfc2r3mNkvmdta+H/5zfdFe6317zmCdhhPYbipaGVFPUZO2cCTgSso28oDvOpCDldo/wl3jUxYNDlwH8LYMqKT3aGaOZr8JbxYzd+L+5GM2KTD+4YRmPtpYS/LWcc3j+fiFXSgX6Mrrgf6ineCRuBxSooDVE+pBakM1U7d5NE25apaAvclzFTmZBg0Sf9e5sgHkR99r9DUeGEQWGNZVUGwti39dFVp+aC9dsA+1/OtNB/HMF5G1MMk9dqvN7n7i9o9Plef2DParn4QU1GhmUKeEiBe4OAmSP+WwD4YvK6iXSKZG6tuTEspw+mR3rK5gBHrEiaNlCtp7O9BnAw4Wjhw== rsa-key-20201218" + ]; + }; + + frankgaa = { + description = "Frank Gaardsted"; + home = "/home/frankgaa"; + group = "frankgaa"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1002; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDTq/IAtLkvHaPKTsp5U9YnhBj7PLFflS9vWpm5e/bFXQkSShkqUOktff1GITIN+RTpUS8zF9UkJA8fj5K382DhIn4jVb9HvQzmHNBTxU5ClpOuKhfibrts5IKMLAiN1enwZYu0iUIVfDKTYmqgAnjN8B6OyzIAB8bsBUMdN29PEwJT4cCVRRySLRfoWiXiZKow71FzXIACgxMwGhj2fpslKQoat2LGny03XR7EZrv36u1OktT28Gxf4ZrGpT9+3SAyf7aW20xHALU/dHXVsfsuqnoqw1InZ5VhvIVtoIj+5Vc5dkTXkychL0Hb+WxiH5O/3T18YUqes08UPZX5G9kB fga@akvaplan.niva.no" + ]; + }; + + hdj002 = { + description = "Hans Kristian Djuve"; + home = "/home/hdj002"; + group = "hdj002"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1003; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAvXTQTRfcpVkaJ+HO6wpgMKO9bdoTvQunbexn1N4jOuJOB5uAKZrZsimucw4DpvzaLZAftLI6RQlmiklxWItZA2UCfhlIZus8wy0cSTic2PkxLUXzBa1wl7nr8anYSK/HReQfTkgSi6LGTsGhejBxe//XC24ygW9eFYZTwOkpD8klNBNFHUA6sXgnzcjT/j3rwUjYI4GVJ82kP3GA0GBDSMwZ45/8ZYBk/Dbja0RJlvTHLSIgAmKzOuor1ORXk4zGdVpsTgNRv7QxeXZT09KJFp9hexMB/fT/OwZopKrHdGrOIIi2vhO6AGh4U+qjnjWY8yn5qHSOAIDDiJQ/iLeCCw== rsa-key-20201217" + ]; + }; + + hes = { + description = "Håvard Espenes"; + home = "/home/hes"; + group = "hes"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1004; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAxOM63++alCeL0dcFdLx5O6YugdXZlcqzNIWgQS42RdhGxHdBzW5Y47KMdbQyp12EAH4kgRaNQ79HkHpDIX9qf4EytVgcWNgZePRnr+sX6vsm1jpeq2jy52CDUIQTmoF8Qh+zGFXuiuYFOFkZ6YokknkEzubfR2w0JMHocD4XS5ZxBn80QVmTL4qH2/1eR2WGhnNdwBI1c1CHDWbKD/iaQAWQMe46jJqHgYRLrZ8ODG8Ucz1/4kZF3B+pmV8gMaaVFC/+ncWuZa3B1zqoOeVbf49qjPKJcCw+5BCo2ZoLPpFQANMvbbth8dHnv4gE7zTK3dHmEPqibsran7++J0LNow== rsa-key-20201217" + ]; + }; + + mad = { + description = "Magnus Drivdal"; + home = "/home/mad"; + group = "mad"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1005; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSHaeXMssp55DPb9Pnqn1WxQqutvOsvIrYDKgTsrnmf/GC3N7h593pX27Qrzjaf0OYHcT7drUbDvRIFsuzmtplff/87fqbx/GGyYUrP+LZwOB7BKHrRlixaVeOOmvE/f9SNM0sIolq47qKFHM2KMsbk5zhSiOmbsCXWzdyYu+kjRgSjSs8NByoKzW9+dUuop/dcTZtDJd0oxUwkpU3c+MSkdAtDpvuxG6vq56Z4j9llg8kzHtLQI7t8w35FhIcXaJOE+LXJfq1FuAl9rw0QMAvotRJ+wHiMOqdkbABkmHG594JNCvZStqN4r8r3ZI20AF8xB3EiLVAuiB08kdEcuoGJCj7IHUwozbds4PO32oDSVy2/IR6VtRJQSNLH99nSbDQ9AmJtn3gSVJvAil1zaw/PZGez/UnDevN7KwUnf6wxH7yxDqgRjEmTu62z7HxQkPKMbuIvcHCQV9ota4eJZFOpr9tH44tuETcvkOButJH1Qa6AMKZPj10bxlqPIHKFS3dhus0BiUIWbjUY9/dC3LwW0S8TeP1NLx0vUeDiqY7BId6jZ8yeguQTvngPFJw5VPEuH5X/6sSXoCn1MnwdhXyeIylE5JoPXahVGQkaHo1Hap0JajxIlsZN6Q719qwSYJouH4mjdFBT46kw8CSH2sedv5qCQPchVmvaADirytYTQ== pckey" + ]; + }; + + peyghamg = { + description = "Peygham Ghaffari Nooran"; + home = "/home/peyghamg"; + group = "peyghamg"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1006; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = []; + }; + + qin = { + description = "Qin Zhou"; + home = "/home/qin"; + group = "qin"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1007; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = []; + }; + + eli = { + description = "Eli Børve"; + home = "/home/eli"; + group = "eli"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1008; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA8cclW3+wlmFl4fNEIqudlrtKVsZCXdzLgXNaGnqZPbqUCVXqw6rigFXSPYnRnE47yi8heC+06ga+4l8LkAuFCPhoZN6xxq6M7qmlNRHtpNn6PrXuDaTkgNig9TD2BCbHGwkqIByPhea4qQbM0mLKRk+a4H0ZGHsJnZI+nm185sSo6jQJ1gCHLk6ZZOoRDGYjoZIs3En3fTSTJBfbluf5A/gVXf2pbvIQ+R5QEOszRcln6nFfTd2Lwu2t63jA1pHN3KQoKcJ9RtBrRC0ZgZrOlEU1bpGLwuHwpj5xoxlsoI49QMgqJL4PE06mNGLYuxl6N/Sj6l90fuPFrEQpf45S0w== rsa-key-20201217" + ]; + }; + + ovanov = { + description = "Øyvind Leikvin"; + home = "/home/ovanov"; + group = "ovanov"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1009; + isNormalUser = true; + createHome = true; + openssh.authorizedKeys.keys = []; + }; + + bast = { + description = "Radovan Bast"; + home = "/home/bast"; + group = "bast"; + extraGroups = [ + "users" + "docker" + ]; + uid = 1010; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFbrEhm1acesXmbgfO5lN1gcTFXqusq61QyCZXunYJpl" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIdcJteh9d/N1o8BbdEMRVxeMjm28saon/Oh2tV0+TYj" + ]; + }; + # @usr@ + }; + +}