Migration to new setup

2019-10-15 15:33:43 +02:00
parent 7b59038e50
commit e4765df729
19 changed files with 346 additions and 3212 deletions
--- a/kube1/k8s.nix
+++ b/kube1/k8s.nix
@@ -0,0 +1,136 @@
+{ pkgs, lib, settings, ...}:
+with lib;
+let
+  cluster-ca = pkgs.stdenv.mkDerivation {
+      name = "cluster-ca";
+      src = ./ca;
+      buildCommand = ''
+        mkdir -p $out
+        cp $src/* $out
+      '';
+  };
+  nixos-kubernetes-join-nodes = workers:
+    let
+      wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
+    in
+    pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
+      #!/bin/sh
+      set -e
+      token=$(cat /var/lib/cfssl/apitoken.secret)
+      for i in ${wrk}; do
+         ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
+      done
+    '';
+  cidr = "10.10.0.0/16";
+in
+rec {
+  kubeMaster = {
+    services.cfssl.ca = "${cluster-ca}/ca.pem";
+    services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
+    services.kubernetes = {
+      roles = [ "master" ];
+      masterAddress = settings.master;
+      apiserverAddress = settings.apiserverAddress;
+      clusterCidr = cidr;
+      kubelet.unschedulable = false;
+      pki.genCfsslCACert = false;
+      pki.caCertPathPrefix = "${cluster-ca}/ca";
+      apiserver = {
+        advertiseAddress = settings.masterAddress;
+        authorizationMode = [ "Node" "RBAC" ];
+        securePort = 8443;
+        insecurePort = 8080;
+        extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
+      };
+      addons = {
+        dns = {
+          enable = true;
+          # clusterDomain = "local";
+          reconcileMode = "EnsureExists";
+        };
+      };
+    };
+    networking.firewall = {
+      allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
+      allowedUDPPorts = [ 53 4053 ];
+    };
+    environment.systemPackages = [
+      pkgs.kubernetes-helm
+      (nixos-kubernetes-join-nodes settings.workers)
+    ];
+  };
+
+  kubeWorker = {
+    services.kubernetes = rec {
+      roles = [ "node" ];
+      clusterCidr = cidr;
+      masterAddress = settings.master;
+      apiserverAddress = settings.apiserverAddress;
+    };
+    networking = {
+      firewall = {
+        enable = true;
+        allowedTCPPorts = [ 4194 10250 ];
+        allowedUDPPorts = [ 53 ];
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
+      };
+    };
+    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
+    virtualisation.docker.autoPrune.enable = true;
+  };
+
+  baseNixos = name: {
+    imports = [
+      (../nixos/hardware-configuration + "/${name}.nix")
+      ../nixos/configuration.nix
+    ];
+    security.pki.certificateFiles = [
+      "${cluster-ca}/ca.pem"
+    ];
+    # services.glusterfs = {
+    #   enable = true;
+    #   # tlsSettings = {
+    #     # caCert = certs.ca.caFile;
+    #     # tlsKeyPath = certs.self.keyFile;
+    #     # tlsPem = certs.self.certFile;
+    #   };
+    # };
+    networking = {
+      hostName = name;
+      extraHosts = settings.clusterHosts;
+      # nameservers = [ masterAddress ];
+      # dhcpcd.extraConfig = ''
+      #  static domain_name_servers=${masterAddress}
+      # '';
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 80 443 111 ];
+      firewall.allowedUDPPorts = [ 111 24007 24008 ];
+    };
+  };
+
+  apiserver = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+        kubeMaster
+      ];
+    };
+
+  worker = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+        kubeWorker
+      ];
+    };
+
+  host = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+      ];
+    };
+}