{ pkgs ? import {} }: let # Pin the deployment package-set to a specific version of nixpkgs # pkgs = import (builtins.fetchTarball { # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; # }) {}; # pkgs = import {}; etcdCluster = import ../etcdCluster.nix; name = "nfs0"; address = "10.255.241.80"; in { nfs0 = { config, pkgs, ... }: with pkgs; { deployment.tags = [ "fs" "fs-data" ]; deployment.targetHost = address; system.autoUpgrade.enable = lib.mkForce false; systemd.targets = { sleep.enable = false; suspend.enable = false; hibernate.enable = false; hybrid-sleep.enable = false; }; environment.etc = { "minio/rootcredentials" = { text = '' accessKey="admin" secretKey="en to tre fire" ''; mode = "600"; uid = 280; }; }; services.udev.extraRules = '' KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" ''; environment.systemPackages = with pkgs; [ rdma-core hwloc ]; cluster.k8sNode = true; features = { host = { inherit address; inherit name; }; os = { externalInterface = "enp33s0f3np3"; nfs.enable = true; nfs.exports = '' /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) ''; }; k8s = { enable = true; node.enable = true; master.enable = false; inherit etcdCluster; }; }; # certs = { # enable = true; # caBundle = ./ca; # certs = [ # { # name = name; # SANs = [ "${name}.cluster.local" address ]; # owner = "nginx"; # group = "nginx"; # } # ]; # }; systemd.services.rc-local = { description = "rc.local script"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; path = [ "/run/current-system/sw/" ]; serviceConfig = { Type = "oneshot"; }; script = '' if [ -e /sys/block/md126 ]; then echo "deadline" > /sys/block/md126/queue/scheduler # echo "4096" > /sys/block/md126/queue/nr_requests echo "4096" > /sys/block/md126/queue/read_ahead_kb echo "always" > /sys/kernel/mm/transparent_hugepage/enabled echo "always" > /sys/kernel/mm/transparent_hugepage/defrag fi grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist ''; }; boot.kernel.sysctl = { "vm.dirty_background_ratio" = 5; "vm.dirty_ratio" = 10; "vm.vfs_cache_pressure" = 50; "vm.min_free_kbytes" = 262144; }; services.minio = { enable = true; region = "store1"; browser = true; rootCredentialsFile = "/etc/minio/rootcredentials"; listenAddress = "0.0.0.0:9000"; dataDir = [ "/data/s3" ]; }; networking = { hostName = name; interfaces.enp33s0f3np3 = { useDHCP = false; ipv4.addresses = [ { address = address; prefixLength = 24; } ]; ipv4.routes = [ { address = "10.255.242.2"; prefixLength = 32; via = "10.255.241.100"; } ]; }; interfaces."ibp65s0" = { useDHCP = false; }; interfaces."ibp65s0.7666" = { useDHCP = false; ipv4.addresses = [ { address = "10.255.243.80"; prefixLength = 24; } ]; }; firewall = { allowedTCPPorts = [ 443 9000 9001 ]; allowedUDPPorts = []; extraCommands = '' iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE ''; }; }; # services.nginx = { # enable = true; # statusPage = false; # virtualHosts = { # "s3ui.oceanbox.io" = { # forceSSL = true; # enableACME = false; # sslTrustedCertificate = "/var/lib/secrets/ca.pem"; # sslCertificate = "/var/lib/secrets/s3.pem"; # sslCertificateKey = "/var/lib/secrets/s3-key.pem"; # serverAliases = []; # locations."/" = { # proxyPass = "http://127.0.0.1:9001"; # extraConfig = '' # allow all; # ''; # }; # }; # }; # }; fileSystems = { "/exports/data" = { device = "/data"; options = [ "bind" ]; }; "/exports/ekman" = { device = "/data/nfs-provisioner"; options = [ "bind" ]; }; "/exports/opt" = { device = "/opt"; options = [ "bind" ]; }; "/vol/local-storage/vol1" = { device = "/vol/vol1"; options = [ "bind" ]; }; }; programs.singularity.enable = true; imports = [ ../cluster.nix ./hardware-configuration.nix ]; }; }