{ pkgs, lib, config, ... }: with lib; let cfg = config.features.host; etcdCluster = { enable = true; existing = true; nodes = { etcd0 = "10.255.241.100"; # etcd1 = "10.255.241.80"; # etcd2 = "10.255.241.81"; }; }; mkSANs = host: [ host.name host.address "127.0.0.1" ]; configuration = { system.autoUpgrade.enable = lib.mkForce false; nixpkgs.overlays = [ (import ./overlays.nix) ]; boot = { loader.systemd-boot.enable = true; loader.efi.canTouchEfiVariables = true; # kernelPackages = pkgs.linuxPackages_5_4; kernelModules = [ "ib_umad" "ib_ipoib" ]; # kernelParams = [ # "console=ttyS0,115200" # "console=tty0" # ]; }; services.udev.extraRules = '' KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x3666" ''; console = { font = "Lat2-Terminus16"; keyMap = "us"; }; i18n = { defaultLocale = "en_US.UTF-8"; extraLocaleSettings = { LC_CTYPE="en_DK.UTF-8"; LC_TIME="en_DK.UTF-8"; LC_PAPER="en_DK.UTF-8"; LC_NAME="en_DK.UTF-8"; LC_ADDRESS="en_DK.UTF-8"; LC_TELEPHONE="en_DK.UTF-8"; LC_MEASUREMENT="en_DK.UTF-8"; LC_IDENTIFICATION="en_DK.UTF-8"; }; }; time.timeZone = "Europe/Oslo"; programs.msmtp = { enable = true; accounts = { default = { auth = false; tls = false; tls_starttls = false; port = 24; from = "ekman@oceanbox.io"; host = "smtpgw.itpartner.no"; # user = "utvikling"; # password = "S0m3rp0m@de#21!"; }; }; defaults = { aliases = "/etc/aliases"; }; }; environment.etc = { "aliases" = { text = '' root: jonas.juselius@oceanbox.io ''; mode = "0644"; }; }; features = { os = { # boot.uefi = true; adminAuthorizedKeys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no" ]; docker.enable = false; }; cachix.enable = false; monitoring.nodeExporter.enable = false; pki = { ca = ./ca; }; hpc = { enable = true; slurm = { client = true; mungeKey = ./munge.key; mungeUid = 995; # hack controlMachine = "ekman"; nodeName = [ "c0-[1-2] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" ]; partitionName = [ "batch Nodes=c0-[1-2] Default=YES MaxTime=INFINITE State=UP" "frontend Nodes=ekman MaxTime=1:00:00 State=UP" ]; }; beegfs = { enable = false; beegfs = { bee0-0 = { mgmtdHost = "mds1-0"; connAuthFile = "/etc/beegfs/connauthfile"; client = { enable = false; mountPoint = "/work"; }; }; }; }; }; k8s = { enable = true; node.enable = true; clusterName = "ekman"; initca = ./ca; cidr = "10.100.0.0/16"; master = { name = "ekman"; address = "10.255.241.100"; extraSANs = [ "ekman.local" "ekman.oceanbox.io" ]; }; ingressNodes = [ "ekman.oceanbox.io" ]; fileserver = "mds1-0"; charts = { acme_email = "innovasjon@itpartner.no"; grafana_smtp_user = "utvikling"; grafana_smtp_password = "S0m3rp0m@de#21!"; }; }; }; services.kubernetes.kubelet.extraSANs = mkSANs { name = cfg.name; address = cfg.address; }; networking = { domain = mkDefault "cluster.local"; defaultGateway = mkDefault "10.255.241.1"; nameservers = mkDefault [ "8.8.8.8" ]; search = mkDefault [ "local" ]; extraHosts = import ./hosts.nix; firewall.extraCommands = '' iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT ''; }; fileSystems = { # "/opt" = { # device = "10.255.241.81:/opt"; # fsType = "nfs"; # options = [ "soft" "rdma" "defaults" "vers=4.2" ]; # }; # "/data" = { # device = "255.241.81:/data"; # fsType = "nfs"; # options = [ "soft" "rdma" "defaults" "vers=4.2" ]; # }; }; environment.variables = {}; systemd.services."serial-getty@ttyS0".enable = true; environment.etc."beegfs/connauthfile" = { source = ./connauthfile; mode = "0400"; uid = 0; gid = 0; }; nix = { maxJobs = 32; trustedUsers = [ "@wheel" ]; binaryCachePublicKeys = [ "ekman.local:2NsTThGkZVJtOs3NVQYjEZ4NLscXlbjqA8Fi7HnAreA=" ]; }; }; shosts = { environment.etc."ssh/shosts.equiv" = { mode = "0644"; uid = 0; gid = 0; text = '' 10.255.241.80 10.255.241.81 '' + builtins.foldl' (a: x: a + "10.255.240.${toString x}\n") "" (builtins.genList (n: n + 100) 17); }; programs.ssh.knownHosts = { ekman = { hostNames = [ "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.8" ]; publicKeyFile = ./pubkeys/ekman.pub; }; } // builtins.foldl' (a: x: let n = toString x; n' = toString (x + 100); in a // { "c0-${n}" = { hostNames = [ "c0-${n}" "c0-${n}.cluster.local" "10.255.241.${n'}" "10.255.243.${n'}" ]; publicKeyFile = ./pubkeys/c0-1.pub; }; }) {} (builtins.genList (n: n) 16); environment.systemPackages = [ openssh-shosts ]; security.wrappers = { ssh-keysign = { source = "${openssh-shosts}/libexec/ssh-keysign"; owner = "root"; group = "root"; permissions = "u+rs,g+rx,o+rx"; }; }; }; openssh-shosts = pkgs.openssh.overrideAttrs (attrs: { buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ]; doCheck = false; # the tests take hours }); myvnc = let myvnc = pkgs.writeScriptBin "myvnc" '' #!${pkgs.runtimeShell} uid=`id -u` port=$((9000+$uid)) shell=`getent passwd $(id -un) | awk -F : '{print $NF}'` # vnc=${pkgs.tigervnc}/bin/vncserver vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver case $1 in -p|--port) shift; port=$1 ;; kill|stop) display=$($vnc -list | sed -n 's/^\(:[0-9]\+\).*/\1/p'| head -1) $vnc -kill $display exit 0 ;; esac ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1 [ $? = 1 ] && $vnc -rfbport $port echo "Xvnc server is running on port $port." exec $shell -i ''; buildCommand = '' mkdir -p $out/bin echo $src > $out/bin/myvnc chmod 755 $out/bin/myvnc ''; in { environment.systemPackages = [ myvnc ]; }; in { options.node = { myvnc = mkEnableOption "Enable myvnc script"; }; config = mkMerge [ configuration shosts (mkIf config.node.myvnc myvnc) ]; imports = [ ../../modules ../../nixos ./users.nix ]; }