platform/nixops/ekman/cluster.nix

{ pkgs, lib, config, ... }:
with lib;
let
  cfg = config.features.host;

  nodes = import ./nodes.nix;

  etcdCluster = {
    enable = true;
    existing = true;
    nodes =
      {
        etcd0 = "10.255.241.100";
        etcd1 = "10.255.241.90";
        etcd2 = "10.255.241.80";
      };
  };

  mkSANs = host: [
    host.name
    host.address
    "127.0.0.1"
  ];

  configuration = {
    system.autoUpgrade.enable = lib.mkForce false;

    nixpkgs.overlays = [
      (import ./overlays.nix)
    ];

    boot = {
      loader.systemd-boot.enable = true;
      loader.efi.canTouchEfiVariables = true;
      # kernelPackages = pkgs.linuxPackages_5_4;
      kernelModules = [ "ib_umad" "ib_ipoib" ];
      # kernelParams = [
      #   "console=ttyS0,115200"
      #   "console=tty0"
      # ];
    };

    # services.udev.extraRules = ''
    #   KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x3666"
    # '';

    console = {
      font = "Lat2-Terminus16";
      keyMap = "us";
    };

    i18n = {
      defaultLocale = "en_US.UTF-8";
      extraLocaleSettings = {
          LC_CTYPE="en_DK.UTF-8";
          LC_TIME="en_DK.UTF-8";
          LC_PAPER="en_DK.UTF-8";
          LC_NAME="en_DK.UTF-8";
          LC_ADDRESS="en_DK.UTF-8";
          LC_TELEPHONE="en_DK.UTF-8";
          LC_MEASUREMENT="en_DK.UTF-8";
          LC_IDENTIFICATION="en_DK.UTF-8";
      };
    };

    time.timeZone = "Europe/Oslo";

    programs.msmtp = {
      enable = true;
      accounts = {
        default = {
          auth = false;
          tls = false;
          tls_starttls = false;
          port = 24;
          from = "ekman@oceanbox.io";
          host = "smtpgw.itpartner.no";
          # user = "utvikling";
          # password = "S0m3rp0m@de#21!";
        };
      };
      defaults = {
        aliases = "/etc/aliases";
      };
    };

    environment.etc = {
      "aliases" = {
        text = ''
          root: jonas.juselius@oceanbox.io
        '';
        mode = "0644";
      };
    };

    features = {
      os = {
        # boot.uefi = true;
        adminAuthorizedKeys = [
          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas"
          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
        ];
        docker.enable = false;
      };
      cachix.enable = false;
      monitoring.nodeExporter.enable = false;
      pki = { ca = ./ca; };
    };

    networking = {
      domain = mkDefault "cluster.local";
      defaultGateway = mkDefault "10.255.241.1";
      nameservers = mkDefault [ "8.8.8.8" ];
      search = mkDefault [ "local" ];
      extraHosts = import ./hosts.nix;
      firewall.extraCommands = ''
        iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
      '';
    };

    environment.variables = {};

    # systemd.services."serial-getty@ttyS0".enable = true;

    environment.etc."beegfs/connauthfile" = {
        source = ./connauthfile;
        mode = "0400";
        uid = 0;
        gid = 0;
      };

    nix = {
      maxJobs = 32;
      trustedUsers = [ "@wheel" ];
      binaryCachePublicKeys = [
        "ekman.local:2NsTThGkZVJtOs3NVQYjEZ4NLscXlbjqA8Fi7HnAreA="
      ];
    };
  };

  compute = {
    features = {
      hpc = {
        enable = true;
        slurm = {
          client = true;
          mungeKey = ./munge.key;
          mungeUid = 996; # hack
          controlMachine = "ekman";
          nodeName = [
            "c0-[1-2] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
            "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
          ];
          partitionName = [
            "batch Nodes=c0-[1-2] Default=YES MaxTime=INFINITE State=UP"
            "frontend Nodes=ekman MaxTime=1:00:00 State=UP"
          ];
        };
        beegfs = {
          enable = false;
          beegfs = {
            bee0-0 = {
              mgmtdHost = "beegfs0";
              connAuthFile = "/etc/beegfs/connauthfile";
              client = {
                enable = false;
                mountPoint = "/work";
              };
            };
          };
        };
      };
    };

    fileSystems = {
      "/opt" = {
        device = "10.255.243.80:/opt";
        fsType = "nfs";
        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
      };
      "/data" = {
        device = "10.255.243.80:/data";
        fsType = "nfs";
        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
      };
    };
  };

  k8s-node = {
    features = {
      k8s = {
        enable = true;
        node.enable = true;
        clusterName = "ekman";
        inherit etcdCluster;
        initca = ./ca;
        cidr = "10.100.0.0/16";
        master = {
          name = "ekman";
          address = "10.255.241.100";
          extraSANs = [ "ekman.local" "ekman.oceanbox.io" ];
        };
        ingressNodes = [
          "ekman.oceanbox.io"
        ];
        fileserver = "mds1-0";
        charts = {
          acme_email = "innovasjon@itpartner.no";
          # grafana_smtp_user = "utvikling";
          # grafana_smtp_password = "S0m3rp0m@de#21!";
        };
      };
    };

    services.kubernetes.kubelet.extraSANs = mkSANs {
      name = cfg.name;
      address = cfg.address;
    };
  };

  shosts = {
    environment.etc."ssh/shosts.equiv" = {
      mode = "0644";
      uid = 0;
      gid = 0;
      text = ''
        10.255.241.80
        10.255.241.90
      '' + builtins.foldl' (a: x: a + "${x.address}\n") "" nodes;
    };

    programs.ssh.knownHosts = {
      ekman = {
        hostNames = [
          "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.8"
        ];
        publicKeyFile = ./pubkeys/ekman.pub;
      };
  } // builtins.foldl' (a: x:
      let
          n = toString x.idx;
          addr = toString (x.idx + 100);
      in a // {
          "c0-${n}" = {
              hostNames = [
                  "c0-${n}"
                  "c0-${n}.cluster.local"
                  "10.255.241.${addr}"
                  "10.255.243.${addr}"
              ];
              publicKeyFile = ./pubkeys/c0-${n}.pub;
          };
        }) {} nodes;

    environment.systemPackages = [ openssh-shosts ];

    security.wrappers = {
        ssh-keysign = {
            source = "${openssh-shosts}/libexec/ssh-keysign";
            owner = "root";
            group = "root";
            permissions = "u+rs,g+rx,o+rx";
        };
    };
  };

  openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {
    buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ];
    doCheck = false; # the tests take hours
  });

in {
  options.cluster = {
    compute = mkEnableOption "Enable compute node configs";
  };

  options.cluster = {
    k8sNode = mkEnableOption "Enable k8s node";
  };

  config = mkMerge [
    configuration
    shosts
    (mkIf config.cluster.compute compute)
    (mkIf config.cluster.k8sNode k8s-node)
  ];

  imports = [
    ../../modules
    ../../nixos
    ./users.nix
  ];
}