From 0e16ea3dbfdea920692803dab0c633e0df876f6f Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Fri, 30 Oct 2020 14:14:36 +0100 Subject: [PATCH] WIP: cluster modules os and fs --- lib/base.nix | 62 ------------------------------ lib/default.nix | 100 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/fs.nix | 64 +++++++++++++++++++++++++++++++ lib/k8s.nix | 22 ----------- lib/module.nix | 33 ---------------- lib/os.nix | 84 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 117 deletions(-) delete mode 100644 lib/base.nix create mode 100644 lib/default.nix create mode 100644 lib/fs.nix delete mode 100644 lib/module.nix create mode 100644 lib/os.nix diff --git a/lib/base.nix b/lib/base.nix deleted file mode 100644 index 9b7e6ca..0000000 --- a/lib/base.nix +++ /dev/null @@ -1,62 +0,0 @@ -{ pkgs, lib, config, ... }: -with lib; -let - cfg = config.k8s; -in -rec { - pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; - - baseNixos = name: - { - users.extraUsers.admin.openssh.authorizedKeys.keys = - cfg.adminAuthorizedKeys; - - boot.kernel.sysctl = { - "kernel.mm.transparent_hugepage.enabled" = "never"; - "net.core.somaxconn" = "512"; - }; - - imports = [ - ./nixos/configuration.nix - ]; - - security.pki.certificateFiles = [ pki.ca.cert ]; - - networking = { - hostName = name; - extraHosts = cfg.clusterHosts; - firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; - firewall.allowedTCPPorts = [ 80 443 111 ]; - firewall.allowedUDPPorts = [ 111 24007 24008 ]; - }; - environment.systemPackages = with pkgs; [ - nfs-utils - ]; - }; - - hostCerts = builtins.foldl' - (a: x: a // { ${x.name} = pki.gencert { - cn = x.name; - ca = x.ca; - o = cfg.clusterName; - }; - }) {} cfg.hosts; - - - mkHost = host: self: { - deployment.targetHost = host.address; - require = [ - (baseNixos host.name) - ]; - }; - - baseDeployment = nodes: attrs: - let - hosts = - builtins.foldl' - (a: x: a // { ${x.name} = mkHost x _; }) {} nodes; - hosts' = lib.recursiveUpdate hosts attrs; - names = builtins.attrNames hosts; - in - builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names; -} diff --git a/lib/default.nix b/lib/default.nix new file mode 100644 index 0000000..1298863 --- /dev/null +++ b/lib/default.nix @@ -0,0 +1,100 @@ +{ pkgs, cfg, lib, config, ... }: +with lib; +let + pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; + + mkCert = host: { + ${host.name} = pki.gencert { + cn = host.name; + ca = cfg.ca; + o = cfg.clusterName; + }; + }; + + # hostCerts = builtins.foldl' + # (a: x: a // { ${x.name} = pki.gencert { + # cn = x.name; + # ca = x.ca; + # o = cfg.clusterName; + # }; + # }) {} cfg.hosts; + + # mkHost = host: self: { + # deployment.targetHost = host.address; + # require = [ + # (baseNixos host.name) + # ]; + # }; + + # baseDeployment = nodes: attrs: + # let + # hosts = + # builtins.foldl' + # (a: x: a // { ${x.name} = mkHost x _; }) {} nodes; + # hosts' = lib.recursiveUpdate hosts attrs; + # names = builtins.attrNames hosts; + # in + # builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names; + +in { + + # k8s = import ./k8s.nix { inherit pgks lib config; }; + + # k8s = rec { + # apiserver = host: self: { + # deployment.targetHost = host.address; + # require = [ + # (os.baseNixos host.name) + # k8s.kubeMaster + # ]; + # }; + + # node = host: self: { + # deployment.targetHost = host.address; + # require = [ + # (os.baseNixos host.name) + # k8s.kubeWorker + # ]; + # }; + + # deployment = masterNode: workerNodes: + # let + # master = { "${master.name}" = apiserver masterNode; }; + # in + # builtins.foldl' (a: x: + # a // { "${x.name}" = mkWorker x; }) master workerNodes; + # }; + + fs = rec { + mkNode = host: self: { + deployment.targetHost = host.address; + + imports = [ host.hw ./fs.nix ]; + + cluster = cfg // { + hostName = host.name; + cert = mkCert host.name; + }; + }; + + mkDeployment = nodes: + builtins.foldl' (a: x: + a // { "${x.name}" = mkNode x; }) {} nodes; + } ; + + # host = rec { + # node = host: self: { + # deployment.targetHost = host.address; + # require = [ + # (os.baseNixos host.name) + # ]; + # }; + + # deployment = masterNode: workerNodes: + # let + # master = { "${master.name}" = apiserver masterNode; }; + # in + # builtins.foldl' (a: x: + # a // { "${x.name}" = mkWorker x; }) master workerNodes; + # }; +} diff --git a/lib/fs.nix b/lib/fs.nix new file mode 100644 index 0000000..68360bc --- /dev/null +++ b/lib/fs.nix @@ -0,0 +1,64 @@ +{ pkgs, lib, config, ... }: +with lib; +let + cfg = config.cluster.fs; + + cert = cfg.cert; + + pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; + + common = { + boot.kernelModules = [ + "dm_snapshot" + "dm_mirror" + "dm_thin_pool" + ]; + + networking = { + firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; + firewall.allowedTCPPorts = [ 111 2049 ]; + firewall.allowedUDPPorts = [ 111 2049 24007 24008 ]; + }; + + environment.systemPackages = [ pkgs.lvm2 ]; + }; + + glusterfs = { + services.glusterfs = { + enable = true; + tlsSettings = { + caCert = pki.ca.cert; + tlsKeyPath = cert.key; + tlsPem = cert.cert; + }; + }; + }; + + nfs = { + services.nfs.server = { + enable = true; + exports = cfg.fs.exports; + }; + }; +in { + options.cluster.fs = { + nfs = { + enable = mkEnableOption "Enable nfs fileserver"; + exports = mkOption { + type = types.str; + default = ""; + }; + }; + glusterfs.enable = mkEnableOption "Enable glusterfs fileserver"; + }; + + config = mkMerge [ + common + + (mkIf cfg.nfs.enable nfs) + + (mkIf cfg.glusterfs.enable glusterfs) + ]; + + imports = [ ./os.nix ]; +} diff --git a/lib/k8s.nix b/lib/k8s.nix index 1f61a1b..a180afe 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -199,27 +199,5 @@ let }; }; - mkApiServer = host: self: - { - deployment.targetHost = host.address; - require = [ - (baseNixos host.name) - kubeMaster - ]; - }; - - mkWorker = host: self: - { - deployment.targetHost = host.address; - require = [ - (baseNixos host.name) - kubeWorker - ]; - }; - - master = { "${settings.master.name}" = mkApiServer settings.master; }; - - deployment = builtins.foldl' (a: x: - a // { "${x.name}" = mkWorker x; }) master settings.workers; in deployment diff --git a/lib/module.nix b/lib/module.nix deleted file mode 100644 index a94cc34..0000000 --- a/lib/module.nix +++ /dev/null @@ -1,33 +0,0 @@ -{ pkgs, config, lib, ... }: -with lib; -let - cfg = config.k8s; -in -{ - options.k8s = { - initca = mkOption { - type = types.path; - }; - - clusterName = mkOption { - type = types.str; - }; - - hosts = mkOption { - type = types.listOf types.set; - default = []; - }; - - clusterHosts = mkOption { - type = types.str; - }; - - adminAuthorizedKeys = mkOption { - type = types.listOf types.str; - default = []; - }; - }; - - config = { - }; -} diff --git a/lib/os.nix b/lib/os.nix new file mode 100644 index 0000000..7397b42 --- /dev/null +++ b/lib/os.nix @@ -0,0 +1,84 @@ +{ pkgs, lib, config, ... }: +with lib; +let + cfg = config.cluster; + + pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; +in +{ + options.cluster = { + initca = mkOption { + type = types.path; + }; + + hostName = mkOption { + type = types.str; + default = null; + }; + + domain = mkOption { + type = types.str; + default = "local"; + }; + + searchDomains = mkOption { + type = types.listOf types.str; + default = [ cfg.domain ]; + }; + + cert = mkOption { + type = types.attrs; + default = null; + }; + + clusterName = mkOption { + type = types.str; + default = null; + }; + + extraHosts = mkOption { + type = types.str; + }; + + adminAuthorizedKeys = mkOption { + type = types.listOf types.str; + default = []; + }; + }; + + config = { + users.extraUsers.admin.openssh.authorizedKeys.keys = + cfg.adminAuthorizedKeys; + + users.extraUsers.root.openssh.authorizedKeys.keys = + cfg.adminAuthorizedKeys; + + networking = { + hostName = cfg.hostName; + domain = cfg.domain; + search = cfg.searchDomains; + extraHosts = cfg.extraHosts; + }; + + security.pki.certificateFiles = [ pki.ca.cert ]; + boot.kernel.sysctl = { + "kernel.mm.transparent_hugepage.enabled" = "never"; + "net.core.somaxconn" = "512"; + }; + + networking = { + firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; + firewall.allowedTCPPorts = [ 80 443 111 ]; + firewall.allowedUDPPorts = [ 111 24007 24008 ]; + }; + + environment.systemPackages = with pkgs; [ + nfs-utils + ]; + }; + + imports = [ + ./nixos/configuration.nix + ]; + +}