diff --git a/certs.nix b/certs.nix index bf4eb8b..e21a050 100644 --- a/certs.nix +++ b/certs.nix @@ -51,7 +51,7 @@ let DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.cluster.local DNS.4 = k8s0-0.itpartner.no - IP.1 = 10.10.10.1 + IP.1 = 10.0.0.1 IP.2 = 10.253.18.100 ''; @@ -65,11 +65,12 @@ let keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [alt_names] - DNS.1 = k8s0-0 - DNS.2 = k8s0-1 - DNS.3 = k8s0-2 - DNS.4 = *.itpartner.no - DNS.5 = *.itpartner.intern + DNS.1 = *.itpartner.no + DNS.2 = *.itpartner.intern + DNS.3 = k8s0-0 + DNS.4 = k8s0-1 + DNS.5 = k8s0-2 + DNS.6 = git01 ''; ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048"; diff --git a/k8s.nix b/k8s.nix index a62f48b..bd114bc 100644 --- a/k8s.nix +++ b/k8s.nix @@ -2,6 +2,7 @@ with import ./certs.nix; let pkgs = import {}; + kube_apiserver = "https://10.253.18.100:443"; etcdServers = [ "etcd0" "etcd1" "etcd2" ]; etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers; etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers; @@ -30,55 +31,38 @@ let systemd.services.flannel.after = [ "etcd.service" ]; }; - kubeConfig = { - systemd.services.kubelet.path = [ pkgs.socat ]; - services.flannel = { - enable = true; - network = "10.10.0.0/16"; - iface = "ens32"; - etcd = { - endpoints = etcdEndpoints; - keyFile = etcd_client_key; - certFile = etcd_client_cert; - caFile = ca_pem; - }; - }; - networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN - systemd.services.docker = { - after = [ "flannel.service" ]; - serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; - }; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU"; - services.kubernetes.etcd = { - servers = etcdEndpoints; - keyFile = etcd_client_key; - certFile = etcd_client_cert; - caFile = ca_pem; - }; - # services.kubernetes.verbose = true; + kubeconfig = { + caFile = ca_pem; + keyFile = worker_key; + certFile = worker_cert; + server = kube_apiserver; }; kubeNode = { services.kubernetes = { roles = [ "node" ]; kubeconfig = { - server = "https://10.253.18.100:4443"; + server = kube_apiserver; keyFile = worker_key; certFile = worker_cert; caFile = ca_pem; }; kubelet = { - tlsKeyFile = worker_key; - tlsCertFile = worker_cert; + enable = true; + clientCaFile = ca_pem; + tlsKeyFile = worker_key; + tlsCertFile = worker_cert; networkPlugin = null; - clusterDns = "10.253.18.100"; + # clusterDns = "10.253.18.100"; + clusterDns = "10.0.0.254"; + inherit kubeconfig; }; }; networking = { firewall = { enable = true; # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; - allowedTCPPorts = [ 53 10250 ]; + allowedTCPPorts = [ 53 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; @@ -89,11 +73,12 @@ let kubeMaster = { services.kubernetes = { roles = [ "master" ]; + kubelet.unschedulable = true; apiserver = { address = "0.0.0.0"; publicAddress = "0.0.0.0"; advertiseAddress = "10.253.18.100"; - securePort = 4443; + securePort = 443; tlsKeyFile = apiserver_key; tlsCertFile = apiserver_cert; clientCaFile = ca_pem; @@ -103,19 +88,40 @@ let serviceAccountKeyFile = apiserver_key; }; scheduler.leaderElect = true; - controllerManager.leaderElect = true; - controllerManager.serviceAccountKeyFile = apiserver_key; - controllerManager.rootCaFile = ca_pem; - dns.enable = true; - dns.port = 4053; + controllerManager = { + leaderElect = true; + serviceAccountKeyFile = apiserver_key; + rootCaFile = ca_pem; + inherit kubeconfig; + }; + addons.dashboard.enable = true; + addons.dns.enable = true; }; networking.firewall = { - allowedTCPPorts = [ 5000 8080 4443 4053 ]; - allowedUDPPorts = [ 4053 ]; + allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ]; + # allowedUDPPorts = [ 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm ]; }; + kubeConfig = { + services.kubernetes = { + verbose = false; + caFile = ca_pem; + flannel.enable = true; + clusterCidr = "10.10.0.0/16"; + etcd = { + servers = etcdEndpoints; + keyFile = etcd_client_key; + certFile = etcd_client_cert; + caFile = ca_pem; + }; + proxy = { + inherit kubeconfig; + }; + }; + }; + baseConfig = node: { imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; networking = { @@ -130,7 +136,7 @@ let }; services.dnsmasq.enable = true; services.dnsmasq.servers = [ - "/cluster.local/10.253.18.100#4053" + "/cluster.local/10.0.0.254#53" ]; }; @@ -142,7 +148,7 @@ let { deployment.targetHost = ip; require = [ base kubeConfig kubeNode ]; - services.kubernetes.dns.enable = false; + services.kubernetes.addons.dns.enable = false; }; in { @@ -172,7 +178,7 @@ in { deployment.targetHost = "10.253.18.101"; require = [ base etcd kubeConfig kubeNode ]; - services.kubernetes.dns.enable = false; + services.kubernetes.addons.dns.enable = false; }; k8s0-2 = { config, lib, pkgs, ... }: @@ -183,6 +189,24 @@ in { deployment.targetHost = "10.253.18.102"; require = [ base etcd kubeConfig kubeNode ]; - services.kubernetes.dns.enable = false; + services.kubernetes.addons.dns.enable = false; + }; + + git01 = { config, lib, pkgs, ... }: + let + base = baseConfig "git01"; + in + { + deployment.targetHost = "10.253.18.103"; + require = [ base kubeConfig kubeNode ]; + services.kubernetes.addons.dns.enable = false; + services.nfs.server = { + enable=true; + exports= '' + /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + networking.firewall.allowedTCPPorts = [ 111 2049 ]; + networking.firewall.allowedUDPPorts = [ 111 2049 ]; }; }