oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Remove docker overlay, update kubernetes

Browse Source
This commit is contained in:
Jonas Juselius
2018-09-26 13:24:24 +02:00
parent 97054258de
commit 249963b4f3
8 changed files with 102 additions and 896 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
nixos/overlays/kubernetes.nix
View File

@@ -5,8 +5,36 @@ with lib;
let
cfg = config.services.kubernetes;
skipAttrs = attrs: map (filterAttrs (k: v: k != "enable"))
(filter (v: !(hasAttr "enable" v) || v.enable) attrs);
# YAML config; see:
# https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
# https://github.com/kubernetes/kubernetes/blob/release-1.10/pkg/kubelet/apis/kubeletconfig/v1beta1/types.go
#
# TODO: migrate the following flags to this config file
#
# --pod-manifest-path
# --address
# --port
# --tls-cert-file
# --tls-private-key-file
# --client-ca-file
# --authentication-token-webhook
# --authentication-token-webhook-cache-ttl
# --authorization-mode
# --healthz-bind-address
# --healthz-port
# --allow-privileged
# --cluster-dns
# --cluster-domain
# --hairpin-mode
# --feature-gates
kubeletConfig = pkgs.runCommand "kubelet-config.yaml" { } ''
echo > $out ${pkgs.lib.escapeShellArg (builtins.toJSON {
kind = "KubeletConfiguration";
apiVersion = "kubelet.config.k8s.io/v1beta1";
${if cfg.kubelet.applyManifests then "staticPodPath" else null} =
manifests;
})}
'';
infraContainer = pkgs.dockerTools.buildImage {
name = "pause";
@@ -15,8 +43,7 @@ let
config.Cmd = "/bin/pause";
};
mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (
builtins.toJSON (
mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON (
let name' =
if name == "kubelet"
then "system:node:${config.services.kubernetes.kubelet.hostname}"
@@ -51,12 +78,14 @@ let
mkKubeConfigOptions = prefix: {
server = mkOption {
description = "${prefix} kube-apiserver server address.";
default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
default = "http://${if cfg.apiserver.advertiseAddress != null
then cfg.apiserver.advertiseAddress
else "127.0.0.1"}:${toString cfg.apiserver.port}";
type = types.str;
};
caFile = mkOption {
description = "${prefix} certificate authrority file used to connect to kube-apiserver.";
description = "${prefix} certificate authority file used to connect to kube-apiserver.";
type = types.nullOr types.path;
default = cfg.caFile;
};
@@ -81,12 +110,18 @@ let
keyFile = mkDefault cfg.kubeconfig.keyFile;
};
cniConfig = pkgs.buildEnv {
name = "kubernetes-cni-config";
paths = imap (i: entry:
pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
) cfg.kubelet.cni.config;
};
cniConfig =
if cfg.kubelet.cni.config != [] && !(isNull cfg.kubelet.cni.configDir) then
throw "Verbatim CNI-config and CNI configDir cannot both be set."
else if !(isNull cfg.kubelet.cni.configDir) then
cfg.kubelet.cni.configDir
else
(pkgs.buildEnv {
name = "kubernetes-cni-config";
paths = imap (i: entry:
pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
) cfg.kubelet.cni.config;
});
manifests = pkgs.buildEnv {
name = "kubernetes-manifests";
@@ -136,7 +171,6 @@ let
'';
in {
###### interface
options.services.kubernetes = {
@@ -223,18 +257,13 @@ in {
type = types.listOf types.str;
};
address = mkOption {
description = "Kubernetes apiserver listening address.";
default = "127.0.0.1";
type = types.str;
};
publicAddress = mkOption {
bindAddress = mkOption {
description = ''
Kubernetes apiserver public listening address used for read only and
secure port.
The IP address on which to listen for the --secure-port port.
The associated interface(s) must be reachable by the rest
of the cluster, and by CLI/web clients.
'';
default = cfg.apiserver.address;
default = "0.0.0.0";
type = types.str;
};
@@ -289,7 +318,7 @@ in {
tokenAuthFile = mkOption {
description = ''
Kubernetes apiserver token authentication file. See
<link xlink:href="https://kubernetes.io/docs/admin/authentication.html"/>
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/>
'';
default = null;
type = types.nullOr types.path;
@@ -298,7 +327,7 @@ in {
basicAuthFile = mkOption {
description = ''
Kubernetes apiserver basic authentication file. See
<link xlink:href="https://kubernetes.io/docs/admin/authentication.html"/>
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/>
'';
default = pkgs.writeText "users" ''
kubernetes,admin,0
@@ -308,22 +337,31 @@ in {
authorizationMode = mkOption {
description = ''
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See
<link xlink:href="https://kubernetes.io/docs/admin/authorization.html"/>
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/>
'';
default = ["RBAC" "Node"];
type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC" "Node"]);
type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]);
};
authorizationPolicy = mkOption {
description = ''
Kubernetes apiserver authorization policy file. See
<link xlink:href="https://kubernetes.io/docs/admin/authorization.html"/>
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/>
'';
default = [];
type = types.listOf types.attrs;
};
webhookConfig = mkOption {
description = ''
Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/>
'';
default = null;
type = types.nullOr types.path;
};
allowPrivileged = mkOption {
description = "Whether to allow privileged containers on Kubernetes.";
default = true;
@@ -342,16 +380,16 @@ in {
runtimeConfig = mkOption {
description = ''
Api runtime configuration. See
<link xlink:href="https://kubernetes.io/docs/admin/cluster-management.html"/>
<link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/>
'';
default = "authentication.k8s.io/v1beta1=true";
example = "api/all=false,api/v1=true";
type = types.str;
};
admissionControl = mkOption {
enableAdmissionPlugins = mkOption {
description = ''
Kubernetes admission control plugins to use. See
Kubernetes admission control plugins to enable. See
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
'';
default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"];
@@ -363,6 +401,15 @@ in {
type = types.listOf types.str;
};
disableAdmissionPlugins = mkOption {
description = ''
Kubernetes admission control plugins to disable. See
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
'';
default = [];
type = types.listOf types.str;
};
serviceAccountKeyFile = mkOption {
description = ''
Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
@@ -583,6 +630,7 @@ in {
type = types.bool;
};
# TODO: remove this deprecated flag
cadvisorPort = mkOption {
description = "Kubernetes kubelet local cadvisor port.";
default = 4194;
@@ -639,6 +687,12 @@ in {
}]
'';
};
configDir = mkOption {
description = "Path to Kubernetes CNI configuration directory.";
type = types.nullOr types.path;
default = null;
};
};
manifests = mkOption {
@@ -792,13 +846,13 @@ in {
path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path;
serviceConfig = {
Slice = "kubernetes.slice";
CPUAccounting = true;
MemoryAccounting = true;
ExecStart = ''${cfg.package}/bin/kubelet \
${optionalString cfg.kubelet.applyManifests
"--pod-manifest-path=${manifests}"} \
${optionalString (taints != "")
"--register-with-taints=${taints}"} \
--kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \
--require-kubeconfig \
--config=${kubeletConfig} \
--address=${cfg.kubelet.address} \
--port=${toString cfg.kubelet.port} \
--register-node=${boolToString cfg.kubelet.registerNode} \
@@ -838,7 +892,7 @@ in {
};
# Allways include cni plugins
services.kubernetes.kubelet.cni.packages = [pkgs.cni];
services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
boot.kernelModules = ["br_netfilter"];
@@ -863,7 +917,7 @@ in {
(mkIf cfg.apiserver.enable {
systemd.services.kube-apiserver = {
description = "Kubernetes Kubelet Service";
description = "Kubernetes APIServer Service";
wantedBy = [ "kubernetes.target" ];
after = [ "network.target" "docker.service" ];
serviceConfig = {
@@ -877,7 +931,7 @@ in {
${optionalString (cfg.etcd.keyFile != null)
"--etcd-keyfile=${cfg.etcd.keyFile}"} \
--insecure-port=${toString cfg.apiserver.port} \
--bind-address=0.0.0.0 \
--bind-address=${cfg.apiserver.bindAddress} \
${optionalString (cfg.apiserver.advertiseAddress != null)
"--advertise-address=${cfg.apiserver.advertiseAddress}"} \
--allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
@@ -905,11 +959,15 @@ in {
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
}"
} \
${optionalString (elem "Webhook" cfg.apiserver.authorizationMode)
"--authorization-webhook-config-file=${cfg.apiserver.webhookConfig}"
} \
--secure-port=${toString cfg.apiserver.securePort} \
--service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
${optionalString (cfg.apiserver.runtimeConfig != "")
"--runtime-config=${cfg.apiserver.runtimeConfig}"} \
--admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
--enable-admission-plugins=${concatStringsSep "," cfg.apiserver.enableAdmissionPlugins} \
--disable-admission-plugins=${concatStringsSep "," cfg.apiserver.disableAdmissionPlugins} \
${optionalString (cfg.apiserver.serviceAccountKeyFile!=null)
"--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
${optionalString cfg.verbose "--v=6"} \
@@ -1000,7 +1058,7 @@ in {
description = "Kubernetes Proxy Service";
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod];
path = [pkgs.iptables pkgs.conntrack_tools];
serviceConfig = {
Slice = "kubernetes.slice";
ExecStart = ''${cfg.package}/bin/kube-proxy \
@@ -1065,6 +1123,7 @@ in {
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
environment.ADDON_PATH = "/etc/kubernetes/addons/";
path = [ pkgs.gawk ];
serviceConfig = {
Slice = "kubernetes.slice";
ExecStart = "${cfg.package}/bin/kube-addons";
@@ -1094,7 +1153,7 @@ in {
];
environment.systemPackages = [ cfg.package ];
users.extraUsers = singleton {
users.users = singleton {
name = "kubernetes";
uid = config.ids.uids.kubernetes;
description = "Kubernetes user";
@@ -1103,7 +1162,7 @@ in {
home = cfg.dataDir;
createHome = true;
};
users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
users.groups.kubernetes.gid = config.ids.gids.kubernetes;
# dns addon is enabled by default
services.kubernetes.addons.dns.enable = mkDefault true;