Migration to new setup

This commit is contained in:
Jonas Juselius
2019-10-15 15:33:43 +02:00
parent 7b59038e50
commit e4765df729
19 changed files with 346 additions and 3212 deletions

View File

@@ -20,7 +20,8 @@
services.openssh.enable = true;
services.nfs.server.enable = true;
services.vmwareGuest.enable = true;
# virtualisation.vmware.guest.enable = true;
programs.fish.enable = true;
programs.tmux.enable = true;
@@ -29,15 +30,15 @@
disabledModules = [
# "services/cluster/kubernetes/default.nix"
"services/cluster/kubernetes/dns.nix"
"services/cluster/kubernetes/dashboard.nix"
# "services/cluster/kubernetes/dns.nix"
# "services/cluster/kubernetes/dashboard.nix"
];
imports = [
./users.nix
./packages.nix
./overlays/dns.nix
./overlays/dashboard.nix
# ./overlays/dns.nix
# ./overlays/dashboard.nix
# ./overlays/kubernetes.nix
];
@@ -53,7 +54,7 @@
];
# The NixOS release to be compatible with for stateful data such as databases.
system.stateVersion = "18.09";
system.stateVersion = "19.03";
system.autoUpgrade.enable = false;
}

View File

@@ -1,330 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.kubernetes.addons.dashboard;
in {
options.services.kubernetes.addons.dashboard = {
enable = mkEnableOption "kubernetes dashboard addon";
rbac = mkOption {
description = "Role-based access control (RBAC) options";
default = {};
type = types.submodule {
options = {
enable = mkOption {
description = "Whether to enable role based access control is enabled for kubernetes dashboard";
type = types.bool;
default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode;
};
clusterAdmin = mkOption {
description = "Whether to assign cluster admin rights to the kubernetes dashboard";
type = types.bool;
default = false;
};
};
};
};
version = mkOption {
description = "Which version of the kubernetes dashboard to deploy";
type = types.str;
default = "v1.8.3";
};
image = mkOption {
description = "Docker image to seed for the kubernetes dashboard container.";
type = types.attrs;
default = {
imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0";
finalImageTag = cfg.version;
sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8";
};
};
tokenTtl = mkOption {
description = "Expiration time (in seconds) of JWE tokens generated by dashboard. Default: 15 min. 0 - never expires.";
type = types.int;
default = 15;
};
};
config = mkIf cfg.enable {
services.kubernetes.kubelet.seedDockerImages = [(pkgs.dockerTools.pullImage cfg.image)];
services.kubernetes.addonManager.addons = {
kubernetes-dashboard-deployment = {
kind = "Deployment";
apiVersion = "apps/v1";
metadata = {
labels = {
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
k8s-app = "kubernetes-dashboard";
version = cfg.version;
"kubernetes.io/cluster-service" = "true";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
name = "kubernetes-dashboard";
namespace = "kube-system";
};
spec = {
replicas = 1;
revisionHistoryLimit = 10;
selector.matchLabels."k8s-app" = "kubernetes-dashboard";
template = {
metadata = {
labels = {
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
k8s-app = "kubernetes-dashboard";
version = cfg.version;
"kubernetes.io/cluster-service" = "true";
};
annotations = {
"scheduler.alpha.kubernetes.io/critical-pod" = "";
};
};
spec = {
priorityClassName = "system-cluster-critical";
containers = [{
name = "kubernetes-dashboard";
image = with cfg.image; "${imageName}:${finalImageTag}";
ports = [{
containerPort = 8443;
protocol = "TCP";
}];
resources = {
limits = {
cpu = "100m";
memory = "300Mi";
};
requests = {
cpu = "100m";
memory = "100Mi";
};
};
args = [
"--auto-generate-certificates"
"--token-ttl=${toString cfg.tokenTtl}"
];
volumeMounts = [{
name = "tmp-volume";
mountPath = "/tmp";
} {
name = "kubernetes-dashboard-certs";
mountPath = "/certs";
}];
livenessProbe = {
httpGet = {
scheme = "HTTPS";
path = "/";
port = 8443;
};
initialDelaySeconds = 30;
timeoutSeconds = 30;
};
}];
volumes = [{
name = "kubernetes-dashboard-certs";
secret = {
secretName = "kubernetes-dashboard-certs";
};
} {
name = "tmp-volume";
emptyDir = {};
}];
serviceAccountName = "kubernetes-dashboard";
tolerations = [{
key = "node-role.kubernetes.io/master";
effect = "NoSchedule";
} {
key = "CriticalAddonsOnly";
operator = "Exists";
}];
};
};
};
};
kubernetes-dashboard-svc = {
apiVersion = "v1";
kind = "Service";
metadata = {
labels = {
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
k8s-app = "kubernetes-dashboard";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "KubeDashboard";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
name = "kubernetes-dashboard";
namespace = "kube-system";
};
spec = {
ports = [{
port = 443;
targetPort = 8443;
}];
selector.k8s-app = "kubernetes-dashboard";
};
};
kubernetes-dashboard-sa = {
apiVersion = "v1";
kind = "ServiceAccount";
metadata = {
labels = {
k8s-app = "kubernetes-dashboard";
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
name = "kubernetes-dashboard";
namespace = "kube-system";
};
};
kubernetes-dashboard-sec-certs = {
apiVersion = "v1";
kind = "Secret";
metadata = {
labels = {
k8s-app = "kubernetes-dashboard";
# Allows editing resource and makes sure it is created first.
"addonmanager.kubernetes.io/mode" = "EnsureExists";
};
name = "kubernetes-dashboard-certs";
namespace = "kube-system";
};
type = "Opaque";
};
kubernetes-dashboard-sec-kholder = {
apiVersion = "v1";
kind = "Secret";
metadata = {
labels = {
k8s-app = "kubernetes-dashboard";
# Allows editing resource and makes sure it is created first.
"addonmanager.kubernetes.io/mode" = "EnsureExists";
};
name = "kubernetes-dashboard-key-holder";
namespace = "kube-system";
};
type = "Opaque";
};
kubernetes-dashboard-cm = {
apiVersion = "v1";
kind = "ConfigMap";
metadata = {
labels = {
k8s-app = "kubernetes-dashboard";
# Allows editing resource and makes sure it is created first.
"addonmanager.kubernetes.io/mode" = "EnsureExists";
};
name = "kubernetes-dashboard-settings";
namespace = "kube-system";
};
};
} // (optionalAttrs cfg.rbac.enable
(let
subjects = [{
kind = "ServiceAccount";
name = "kubernetes-dashboard";
namespace = "kube-system";
}];
labels = {
k8s-app = "kubernetes-dashboard";
k8s-addon = "kubernetes-dashboard.addons.k8s.io";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
in
(if cfg.rbac.clusterAdmin then {
kubernetes-dashboard-crb = {
apiVersion = "rbac.authorization.k8s.io/v1";
kind = "ClusterRoleBinding";
metadata = {
name = "kubernetes-dashboard";
inherit labels;
};
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "ClusterRole";
name = "cluster-admin";
};
inherit subjects;
};
}
else
{
# Upstream role- and rolebinding as per:
# https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/alternative/kubernetes-dashboard.yaml
kubernetes-dashboard-role = {
apiVersion = "rbac.authorization.k8s.io/v1";
kind = "Role";
metadata = {
name = "kubernetes-dashboard-minimal";
namespace = "kube-system";
inherit labels;
};
rules = [
# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
{
apiGroups = [""];
resources = ["secrets"];
verbs = ["create"];
}
# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
{
apiGroups = [""];
resources = ["configmaps"];
verbs = ["create"];
}
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
{
apiGroups = [""];
resources = ["secrets"];
resourceNames = ["kubernetes-dashboard-key-holder"];
verbs = ["get" "update" "delete"];
}
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
{
apiGroups = [""];
resources = ["configmaps"];
resourceNames = ["kubernetes-dashboard-settings"];
verbs = ["get" "update"];
}
# Allow Dashboard to get metrics from heapster.
{
apiGroups = [""];
resources = ["services"];
resourceNames = ["heapster"];
verbs = ["proxy"];
}
{
apiGroups = [""];
resources = ["services/proxy"];
resourceNames = ["heapster" "http:heapster:" "https:heapster:"];
verbs = ["get"];
}
];
};
kubernetes-dashboard-rb = {
apiVersion = "rbac.authorization.k8s.io/v1";
kind = "RoleBinding";
metadata = {
name = "kubernetes-dashboard-minimal";
namespace = "kube-system";
inherit labels;
};
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "Role";
name = "kubernetes-dashboard-minimal";
};
inherit subjects;
};
})
));
};
}

View File

@@ -1,329 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
version = "1.2.5";
cfg = config.services.kubernetes.addons.dns;
ports = {
dns = 10053;
health = 10054;
metrics = 10055;
};
in {
options.services.kubernetes.addons.dns = {
enable = mkEnableOption "kubernetes dns addon";
clusterIp = mkOption {
description = "Dns addon clusterIP";
# this default is also what kubernetes users
default = (
concatStringsSep "." (
take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
))
) + ".254";
type = types.str;
};
clusterDomain = mkOption {
description = "Dns cluster domain";
default = "cluster.local";
type = types.str;
};
replicas = mkOption {
description = "Number of DNS pod replicas to deploy in the cluster.";
default = 2;
type = types.int;
};
reconcileMode = mkOption {
description = ''
Controls the addon manager reconciliation mode for the DNS addon.
See: <link xlink:href="https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md"/>
'';
default = "Reconcile";
type = types.enum [ "Reconcile" "EnsureExists" ];
};
coredns = mkOption {
description = "Docker image to seed for the CoreDNS container.";
type = types.attrs;
default = {
imageName = "coredns/coredns";
imageDigest = "sha256:33c8da20b887ae12433ec5c40bfddefbbfa233d5ce11fb067122e68af30291d6";
finalImageTag = version;
sha256 = "13q19rgwapv27xcs664dw502254yw4zw63insf6g2danidv2mg6i";
};
};
};
config = mkIf cfg.enable {
services.kubernetes.kubelet.seedDockerImages =
singleton (pkgs.dockerTools.pullImage cfg.coredns);
services.kubernetes.addonManager.addons = {
coredns-sa = {
apiVersion = "v1";
kind = "ServiceAccount";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
};
name = "coredns";
namespace = "kube-system";
};
};
coredns-cr = {
apiVersion = "rbac.authorization.k8s.io/v1beta1";
kind = "ClusterRole";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/bootstrapping" = "rbac-defaults";
};
name = "system:coredns";
};
rules = [
{
apiGroups = [ "" ];
resources = [ "endpoints" "services" "pods" "namespaces" ];
verbs = [ "list" "watch" ];
}
{
apiGroups = [ "" ];
resources = [ "nodes" ];
verbs = [ "get" ];
}
];
};
coredns-crb = {
apiVersion = "rbac.authorization.k8s.io/v1beta1";
kind = "ClusterRoleBinding";
metadata = {
annotations = {
"rbac.authorization.kubernetes.io/autoupdate" = "true";
};
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/bootstrapping" = "rbac-defaults";
};
name = "system:coredns";
};
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "ClusterRole";
name = "system:coredns";
};
subjects = [
{
kind = "ServiceAccount";
name = "coredns";
namespace = "kube-system";
}
];
};
coredns-cm = {
apiVersion = "v1";
kind = "ConfigMap";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
};
name = "coredns";
namespace = "kube-system";
};
data = {
Corefile = ".:${toString ports.dns} {
errors
health :${toString ports.health}
kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}
prometheus :${toString ports.metrics}
proxy . /etc/resolv.conf
cache 30
loop
reload
loadbalance
}";
};
};
coredns-deploy = {
apiVersion = "extensions/v1beta1";
kind = "Deployment";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "CoreDNS";
};
name = "coredns";
namespace = "kube-system";
};
spec = {
replicas = cfg.replicas;
selector = {
matchLabels = { k8s-app = "kube-dns"; };
};
strategy = {
rollingUpdate = { maxUnavailable = 1; };
type = "RollingUpdate";
};
template = {
metadata = {
labels = {
k8s-app = "kube-dns";
};
};
spec = {
containers = [
{
args = [ "-conf" "/etc/coredns/Corefile" ];
image = with cfg.coredns; "${imageName}:${finalImageTag}";
imagePullPolicy = "Never";
livenessProbe = {
failureThreshold = 5;
httpGet = {
path = "/health";
port = ports.health;
scheme = "HTTP";
};
initialDelaySeconds = 60;
successThreshold = 1;
timeoutSeconds = 5;
};
name = "coredns";
ports = [
{
containerPort = ports.dns;
name = "dns";
protocol = "UDP";
}
{
containerPort = ports.dns;
name = "dns-tcp";
protocol = "TCP";
}
{
containerPort = ports.metrics;
name = "metrics";
protocol = "TCP";
}
];
resources = {
limits = {
memory = "170Mi";
};
requests = {
cpu = "100m";
memory = "70Mi";
};
};
securityContext = {
allowPrivilegeEscalation = false;
capabilities = {
drop = [ "all" ];
};
readOnlyRootFilesystem = true;
};
volumeMounts = [
{
mountPath = "/etc/coredns";
name = "config-volume";
readOnly = true;
}
];
}
];
dnsPolicy = "Default";
nodeSelector = {
"beta.kubernetes.io/os" = "linux";
};
serviceAccountName = "coredns";
tolerations = [
{
effect = "NoSchedule";
key = "node-role.kubernetes.io/master";
}
{
key = "CriticalAddonsOnly";
operator = "Exists";
}
];
volumes = [
{
configMap = {
items = [
{
key = "Corefile";
path = "Corefile";
}
];
name = "coredns";
};
name = "config-volume";
}
];
};
};
};
};
coredns-svc = {
apiVersion = "v1";
kind = "Service";
metadata = {
annotations = {
"prometheus.io/port" = toString ports.metrics;
"prometheus.io/scrape" = "true";
};
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "CoreDNS";
};
name = "kube-dns";
namespace = "kube-system";
};
spec = {
clusterIP = cfg.clusterIp;
ports = [
{
name = "dns";
port = 53;
targetPort = ports.dns;
protocol = "UDP";
}
{
name = "dns-tcp";
port = 53;
targetPort = ports.dns;
protocol = "TCP";
}
];
selector = { k8s-app = "kube-dns"; };
};
};
};
services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
};
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +0,0 @@
self: super: {
super.config.services.kubernetes.addons.dns =
super.callPackage ./dns.nix {};
super.config.services.kubernetes.addons.dashboard =
super.callPackage ./dashboard.nix {};
}

View File

@@ -33,6 +33,7 @@
ebtables
ethtool
socat
docker
];
in
k8s ++