diff --git a/clusters/ekman/cluster.nix b/clusters/ekman/cluster.nix index 7b0207e..4836042 100644 --- a/clusters/ekman/cluster.nix +++ b/clusters/ekman/cluster.nix @@ -12,17 +12,24 @@ let configuration = { system.autoUpgrade.enable = lib.mkForce false; + nixpkgs.overlays = [ + (import ./overlays.nix) + ]; + boot = { loader.systemd-boot.enable = true; loader.efi.canTouchEfiVariables = true; kernelPackages = pkgs.linuxPackages_5_4; kernelModules = [ "ib_umad" "ib_ipoib" ]; - kernelParams = [ - "console=ttyS0,115200" - "console=tty0" - ]; + # kernelParams = [ + # "console=ttyS0,115200" + # "console=tty0" + # ]; }; + services.udev.extraRules = '' + KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x3666" + ''; console = { font = "Lat2-Terminus16"; @@ -45,6 +52,34 @@ let time.timeZone = "Europe/Oslo"; + programs.msmtp = { + enable = true; + accounts = { + default = { + auth = false; + tls = false; + tls_starttls = false; + port = 24; + from = "ekman@oceanbox.io"; + host = "smtpgw.itpartner.no"; + # user = "utvikling"; + # password = "S0m3rp0m@de#21!"; + }; + }; + defaults = { + aliases = "/etc/aliases"; + }; + }; + + environment.etc = { + "aliases" = { + text = '' + root: jonas.juselius@oceanbox.io + ''; + mode = "0644"; + }; + }; + features = { os = { # boot.uefi = true; @@ -54,14 +89,6 @@ let "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no" ]; docker.enable = false; - - mailRelay = { - enable = true; - adminEmail = "jonas.juselius@tromso.serit.no"; - mailDomain = "itpartner.no"; - mailGateway = "smtpgw.itpartner.no:465"; - mailAuthUser = "utvikling"; - }; }; cachix.enable = false; @@ -89,9 +116,9 @@ let beegfs = { bee0-0 = { mgmtdHost = "bee0-0"; - connAuthFile = ""; + connAuthFile = "/etc/beegfs/connauthfile"; client = { - enable = true; + enable = false; mountPoint = "/work"; }; }; @@ -107,11 +134,11 @@ let cidr = "10.100.0.0/16"; master = { name = "ekman"; - address = "10.255.240.200"; - extraSANs = [ "ekman.local" ]; + address = "10.255.241.8"; + extraSANs = [ "ekman.local" "ekman.oceanbox.io" ]; }; ingressNodes = [ - "ekman.local" + "ekman.oceanbox.io" ]; fileserver = "bee0-0"; charts = { @@ -128,13 +155,13 @@ let }; networking = { - domain = mkDefault "oceanbox.io"; - defaultGateway = mkDefault "10.1.61.1"; + domain = mkDefault "cluster.local"; + defaultGateway = mkDefault "10.255.241.1"; nameservers = mkDefault [ "8.8.8.8" ]; search = mkDefault [ "local" ]; extraHosts = import ./hosts.nix; firewall.extraCommands = '' - iptables -I INPUT -s 10.255.240.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT ''; }; @@ -142,12 +169,19 @@ let systemd.services."serial-getty@ttyS0".enable = true; + environment.etc."beegfs/connauthfile" = { + source = ./connauthfile; + mode = "0400"; + uid = 0; + gid = 0; + }; + nix = { maxJobs = 32; trustedUsers = [ "@wheel" ]; - # binaryCachePublicKeys = [ - # "ekman-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig==" - # ]; + binaryCachePublicKeys = [ + "ekman:pka41J3q4j9ZC3dr4y+sDN9uMW0pAxoWeCkrzUlqcZs=" + ]; }; }; @@ -155,40 +189,25 @@ let deployment.targetHost = cfg.address; }; - # i40efix = { - # boot = let kernelExtras = pkgs.callPackage ./kernel.nix { - # kernel = pkgs.linuxPackages_5_4.kernel; - # }; in { - # extraModulePackages = [ kernelExtras.i40e2 ]; - # kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ]; - # }; - # }; - i40efix = { - boot = { - extraModulePackages = []; - kernelModules = [ "ib_umad" "ib_ipoib" ]; - }; - }; - shosts = { environment.etc."ssh/shosts.equiv" = { mode = "0644"; uid = 0; gid = 0; text = '' - 10.255.240.200 - 10.255.240.201 + 10.255.241.8 + 10.255.241.11 ''; }; programs.ssh.knownHosts = { ekman = { hostNames = [ - "ekman" "ekman.local" "ekman.oceanbox.io" "10.255.240.200" + "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.8" ]; publicKeyFile = ./pubkeys/ekman.pub; }; - c0-1 = { hostNames = [ "c0-1" "c0-1.local" "10.255.240.201" "10.255.241.201" ]; publicKeyFile = ./pubkeys/c0-1.pub; }; + c0-1 = { hostNames = [ "c0-1" "c0-1.cluster.local" "10.255.241.11" "10.255.243.11" ]; publicKeyFile = ./pubkeys/c0-1.pub; }; }; environment.systemPackages = [ openssh-shosts ]; @@ -205,6 +224,7 @@ let openssh-shosts = pkgs.openssh.overrideAttrs (attrs: { buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ]; + doCheck = false; # the tests take hours }); myvnc = @@ -243,8 +263,6 @@ let in { options.node = { - i40efix = mkEnableOption "Apply fix for i40e driver"; - myvnc = mkEnableOption "Enable myvnc script"; }; @@ -255,8 +273,6 @@ in { shosts - (mkIf config.node.i40efix i40efix) - (mkIf config.node.myvnc myvnc) ]; diff --git a/clusters/ekman/connauthfile b/clusters/ekman/connauthfile new file mode 100644 index 0000000..37cd965 --- /dev/null +++ b/clusters/ekman/connauthfile @@ -0,0 +1 @@ +ä·q‹u \ No newline at end of file diff --git a/clusters/ekman/default.nix b/clusters/ekman/default.nix index cb9143a..c92b91f 100644 --- a/clusters/ekman/default.nix +++ b/clusters/ekman/default.nix @@ -7,20 +7,21 @@ let pkgs = import {}; etcdNodes = { - c0-0 = "10.255.240.200"; - c0-1 = "10.255.240.201"; + ekman = "10.255.241.8"; + nsf0-0 = "10.255.241.9"; + bee0-0 = "10.255.241.10"; }; etcdCluster = { enable = true; - existing = false; + existing = true; nodes = etcdNodes; }; nodes = with builtins; let nodes = genList (n: n + 1) 1; in - map (n: ({ name = "c0-${toString n}"; address = "10.255.240.20${toString n}"; })) nodes; + map (n: ({ name = "c0-${toString n}"; address = "10.255.241.${toString (n + 10)}"; })) nodes; ekman = { # deployment.tags = [ "frontend" ]; @@ -35,16 +36,16 @@ let features = { host = { - address = "10.255.240.200"; - name = "c0-0"; + address = "10.255.241.8"; + name = "ekman"; }; os = { - externalInterface = "eno1"; + externalInterface = "enp33s0f0np0"; nfs.enable = true; nfs.exports = '' - /exports 10.255.240.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) ''; }; @@ -106,28 +107,28 @@ let interfaces.enp33s0f0np0 = { useDHCP = false; ipv4.addresses = [ { - address = "10.255.240.200"; + address = "10.255.241.8"; prefixLength = 24; } ]; }; - # interfaces.enp33s0f0np1 = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = "10.1.61.100"; - # prefixLength = 24; - # } ]; - # }; - # interfaces.ibp59s0 = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = "10.255.241.200"; - # prefixLength = 24; - # } ]; + interfaces.enp33s0f0np1 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.242.2"; + prefixLength = 24; + } ]; }; - defaultGateway = "10.255.240.1"; + interfaces."ibp1s0.3666" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.8"; + prefixLength = 24; + } ]; + }; + defaultGateway = "10.255.241.1"; firewall.extraCommands = '' - iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT - iptables -t nat -A POSTROUTING -s 10.255.241.0/24 -j MASQUERADE + iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE ''; }; @@ -161,7 +162,7 @@ let }; nix.extraOptions = '' - secret-key-files = /etc/nix/ekman.private + secret-key-files = /etc/nix/ekman.key ''; services.xserver = { @@ -255,7 +256,7 @@ let fileSystems = { "/frontend" = { - device = "10.255.240.200:/home"; + device = "10.255.241.8:/home"; fsType = "nfs"; options = [ "soft" @@ -286,7 +287,7 @@ let mkCompute = host: let - ipoib = builtins.replaceStrings [".240."] [".241."] host.address; + ipoib = builtins.replaceStrings [".241."] [".243."] host.address; hw = ./hw + "/${host.name}.nix"; in { "${host.name}" = { @@ -298,7 +299,6 @@ let }; node = { - i40efix = true; }; networking = { @@ -309,20 +309,20 @@ let address = host.address; prefixLength = 24; } ]; - # ipv4.routes = [ { - # address = "10.1.62.2"; - # prefixLength = 32; - # via = "10.1.61.100"; - # } ]; + ipv4.routes = [ { + address = "10.255.242.2"; + prefixLength = 32; + via = "10.1.241.8"; + } ]; }; - # interfaces.ibp65s0 = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = ipoib; - # prefixLength = 24; - # } ]; - # }; + interfaces."ibp1s0.3666" = { + useDHCP = false; + ipv4.addresses = [ { + address = ipoib; + prefixLength = 24; + } ]; + }; }; imports = [ ./cluster.nix hw ]; } diff --git a/clusters/ekman/hosts.nix b/clusters/ekman/hosts.nix index ccfc6dc..7cbfbee 100644 --- a/clusters/ekman/hosts.nix +++ b/clusters/ekman/hosts.nix @@ -1,10 +1,11 @@ '' - 10.255.240.200 frontend frontend.local c0-0 ekman ekman.oceanbox.io - 10.255.240.201 c0-1 c0-1.local + 10.255.240.200 ekman ekman.cluster.local + 10.255.240.200 etcd0 etcd0.cluster.local + 10.255.240.201 c0-1 c0-1.cluster.local - # 10.1.61.80 bee0-0 bee0-0.local + # 10.1.61.80 bee0-0 bee0-0.cluster.local - # 10.1.63.101 ib0-1 ib0-1.local + # 10.1.63.101 ib0-1 ib0-1.cluster.local - # 10.1.63.80 ibmds0-0 ibmds0-0.local + # 10.1.63.80 ibmds0-0 ibmds0-0.cluster.local '' diff --git a/clusters/ekman/overlays.nix b/clusters/ekman/overlays.nix new file mode 100644 index 0000000..ea76aba --- /dev/null +++ b/clusters/ekman/overlays.nix @@ -0,0 +1,11 @@ +self: super: +let + msmtp = super.msmtp.overrideAttrs (attrs: rec { + configureFlags = attrs.configureFlags ++ [ "--with-tls=openssl" ]; + buildInputs = attrs.buildInputs ++ [ super.openssl ]; + }); +in +{ + # inherit msmtp; +} + diff --git a/clusters/ekman/users.nix b/clusters/ekman/users.nix index e16f322..df9fec6 100644 --- a/clusters/ekman/users.nix +++ b/clusters/ekman/users.nix @@ -1,7 +1,6 @@ { pkgs, ... }: { users.groups = { - admin = { gid = 10000; }; jonas = { gid = 1000; }; olean = { gid = 1001; }; frankgaa = { gid = 1002; }; @@ -21,35 +20,6 @@ }; users.users = { - admin = { - description = "Administrator"; - home = "/home/admin"; - group = "admin"; - extraGroups = [ - "users" - "wheel" - "root" - "adm" - "admin" - "cdrom" - "fuse" - "wireshark" - "libvirtd" - "networkmanager" - "tty" - "keys" - ]; - uid = 10000; - isNormalUser = true; - createHome = false; - useDefaultShell = false; - shell = pkgs.fish; - openssh.authorizedKeys.keys = [ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas-3" - ]; - }; - jonas = { description = "Jonas Juselius"; home = "/home/jonas";