oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add Kubernetes, remove nginx

Browse Source
This commit is contained in:
Jonas Juselius
2021-10-06 11:03:30 +02:00
parent a0e109077f
commit 9bf1722b03
5 changed files with 228 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
clusters/stokes/default.nix
View File

@@ -1,6 +1,29 @@
let
# Pin the deployment package-set to a specific version of nixpkgs
# pkgs = import (builtins.fetchTarball {
# url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
# sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
# }) {};
pkgs = import <nixpkgs> {};
etcdNodes = {
c0-0 = "10.1.61.100";
c0-1 = "10.1.61.101";
c0-2 = "10.1.61.102";
};
etcdCluster = {
enable = true;
existing = true;
nodes = etcdNodes;
};
nodes =
with builtins;
let nodes = genList (n: n + 1) 8; in
map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;
stokes = {
node.address = "10.1.62.2";
node.myvnc = true;
systemd.targets = {
@@ -11,6 +34,11 @@ let
};
features = {
host = {
address = "10.1.62.2";
name = "c0-0";
};
os = {
externalInterface = "eno1";
nfs.enable = true;
@@ -25,13 +53,17 @@ let
frontend = true;
};
k8s = {
master.enable = true;
node.enable = true;
inherit nodes;
inherit etcdCluster;
};
monitoring = {
server = {
enable = true;
scrapeHosts = [
"frontend" "mds0-0"
"c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8"
];
enable = false;
scrapeHosts = [ "frontend" "mds0-0" ] ++ (builtins.map (x: x.name) nodes);
defaultAlertReceiver = {
email_configs = [
{ to = "jonas.juselius@tromso.serit.no"; }
@@ -48,7 +80,7 @@ let
];
};
};
webUI.enable = true;
webUI.enable = false;
webUI.acmeEmail = "innovasjon@itpartner.no";
webUI.allow = [
"10.1.2.0/24"
@@ -120,6 +152,14 @@ let
device = "10.1.63.80:/data";
fsType = "nfs";
};
"/vol/local-storage/vol1" = {
device = "/vol/vol1";
options = [ "bind" ];
};
"/vol/local-storage/vol2" = {
device = "/vol/vol2";
options = [ "bind" ];
};
};
security.pam.services.sshd.googleAuthenticator.enable = true;
@@ -169,16 +209,10 @@ let
};
};
imports = [ ./cluster.nix ./hw/frontend.nix ];
};
compute = {
features = {
os.externalInterface = "eno33";
hpc.compute = true;
};
fileSystems = {
"/home/stokes" = {
device = "10.1.63.100:/home";
@@ -195,49 +229,50 @@ let
};
};
genComputeNodes = idx: nNodes:
let
nodeList = builtins.genList (x: x + 1) nNodes;
mkCompute = n:
let
ip = "10.1.61.${toString (n + 100)}";
ipoib = "10.1.63.${toString (n + 100)}";
name = "c${toString idx}-${toString n}";
hw = ./hw + "/${name}.nix";
in {
"${name}" = {
node = {
address = ip;
i40efix = true;
};
networking = {
useDHCP = false;
interfaces.eno33 = {
useDHCP = false;
ipv4.addresses = [ {
address = ip;
prefixLength = 24;
} ];
ipv4.routes = [ {
address = "10.1.62.2";
prefixLength = 32;
via = "10.1.61.100";
} ];
mkCompute = host:
let
ipoib = builtins.replaceStrings [".61."] [".63."] host.address;
hw = ./hw + "/${host.name}.nix";
in {
"${host.name}" = {
features = {
inherit host;
os.externalInterface = "eno33";
hpc.compute = true;
k8s = { inherit etcdCluster; };
};
node = {
i40efix = true;
};
networking = {
useDHCP = false;
interfaces.eno33 = {
useDHCP = false;
ipv4.addresses = [ {
address = host.address;
prefixLength = 24;
} ];
ipv4.routes = [ {
address = "10.1.62.2";
prefixLength = 32;
via = "10.1.61.100";
} ];
};
interfaces.ibp65s0 = {
useDHCP = false;
ipv4.addresses = [ {
address = ipoib;
prefixLength = 24;
} ];
};
};
imports = [ ./cluster.nix hw ];
} // compute;
};
in
builtins.foldl' (a: n: a // mkCompute n) {} nodeList;
interfaces.ibp65s0 = {
useDHCP = false;
ipv4.addresses = [ {
address = ipoib;
prefixLength = 24;
} ];
};
};
imports = [ ./cluster.nix hw ];
}
// compute;
};
in
{ inherit stokes; } // genComputeNodes 0 8
{ inherit stokes; } // builtins.foldl' (a: n: a // mkCompute n) {} nodes