WIP: Restructuring to new scheme

clusters/fs0/build.nix

@@ -0,0 +1,2 @@
+with import <nixpkgs> {};
+pkgs.callPackage ./certs.nix {}

clusters/fs0/certs.nix

@@ -0,0 +1,11 @@
+{ pkgs, ...}:
+let
+  pki = pkgs.callPackage ../lib/pki.nix {};
+in
+{
+    initca = pki.initca;
+    ca = pki.ca;
+    fs0-0 = pki.trust "fs0-0" ''"10.253.18.106", "127.0.0.1"'';
+    fs0-1 = pki.trust "fs0-1" ''"10.1.2.164", "127.0.0.1"'';
+}
+

clusters/fs0/deployment.nix

@@ -0,0 +1,68 @@
+with import <nixpkgs> {};
+let
+  pki = pkgs.callPackage ../lib/pki.nix {};
+  certs = pkgs.callPackage ./certs.nix {};
+  clusterHosts = ''
+    10.253.18.106 fs0-0 fs0-0.local
+    10.1.2.164    fs0-1 fs0-1.local
+    10.253.18.100 fs0-2 fs0-2.local
+  '';
+
+  nixosConfig = node:
+    let
+      cert = pki.toSet certs.${node};
+    in
+    {
+      imports = [
+        (../nixos/hardware-configuration + "/${node}.nix")
+        ../nixos/configuration.nix
+      ];
+      boot.kernelModules = [
+        "dm_snapshot"
+        "dm_mirror"
+        "dm_thin_pool"
+      ];
+      networking = {
+        hostName = node;
+        extraHosts = clusterHosts;
+        firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+        firewall.allowedTCPPorts = [ 111 ];
+        firewall.allowedUDPPorts = [ 111 24007 24008 ];
+      };
+      services.glusterfs = {
+        enable = true;
+        tlsSettings = {
+          caCert = pki.ca.cert;
+          tlsKeyPath = cert.key;
+          tlsPem = cert.cert;
+        };
+      };
+      environment.systemPackages = [ pkgs.lvm2 ];
+    };
+in
+{
+  fs0-0 = { ... }:
+    let
+      base = nixosConfig "fs0-0";
+    in
+    {
+      deployment.targetHost = "10.253.18.106";
+      require = [ base ];
+      services.nfs.server = {
+        enable=true;
+        exports= ''
+          /vol/brick0/nfs0 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+        '';
+      };
+      networking.firewall.allowedTCPPorts = [ 111 2049 ];
+      networking.firewall.allowedUDPPorts = [ 111 2049 ];
+    };
+  fs0-1 = { ... }:
+    let
+      base = nixosConfig "fs0-1";
+    in
+    {
+      deployment.targetHost = "10.1.2.164";
+      require = [ base ];
+    };
+}

clusters/fs0/fs0-0.nix

@@ -0,0 +1,29 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/6b3d4c49-9719-49b3-8210-d53374cd0eff";
+      fsType = "ext4";
+    };
+  fileSystems."/var/log" =
+    { device = "/dev/disk/by-uuid/c1e78683-4fde-4029-a9f3-7631df649b2f";
+      fsType = "ext4";
+    };
+  fileSystems."/vol/brick0" =
+    { device = "/dev/gfs_vg/brick0";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/fs0/fs0-1.nix

@@ -0,0 +1,30 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/e8820516-9c21-46f4-9dde-a7a77bf67bbd";
+      fsType = "ext4";
+    };
+  fileSystems."/var/log" =
+    { device = "/dev/disk/by-uuid/c130b88c-0699-4836-b967-47bdee0a5453";
+      fsType = "ext4";
+    };
+  fileSystems."/vol/brick0" =
+    { device = "/dev/gfs_vg/brick0";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
+

clusters/kube0/build.nix

@@ -0,0 +1,2 @@
+with import <nixpkgs> {};
+pkgs.callPackage ./certs.nix {}

clusters/kube0/certs.nix

@@ -0,0 +1,36 @@
+{ pkgs, ...}:
+let
+  pki = pkgs.callPackage ../lib/pki.nix {};
+in
+{
+  initca = pki.initca;
+  ca = pki.ca;
+  apiserver = pki.apiserver ''
+        "10.253.18.100",
+        "10.0.0.1",
+        "127.0.0.1",
+        "kubernetes",
+        "kubernetes.default.svc",
+        "etcd0",
+        "fs0-2",
+        "k0-0"
+      '';
+  kube-proxy = pki.kube-proxy;
+  admin = pki.admin;
+  etcd = pki.etcd ''
+      "etcd0",
+      "etcd1",
+      "etcd2",
+      "10.253.18.100",
+      "10.253.18.101",
+      "10.253.18.102",
+      "127.0.0.1"
+    '';
+  k0-0 = pki.worker { name = "k0-0"; ip = "10.253.18.100"; };
+  k0-1 = pki.worker { name = "k0-1"; ip = "10.253.18.101"; };
+  k0-2 = pki.worker { name = "k0-2"; ip = "10.253.18.102"; };
+  k0-3 = pki.worker { name = "k0-3"; ip = "10.253.18.103"; };
+  k0-4 = pki.worker { name = "k0-4"; ip = "10.253.18.107"; };
+  k0-5 = pki.worker { name = "k0-5"; ip = "10.253.18.108"; };
+}
+

clusters/kube0/deployment.nix

@@ -0,0 +1,89 @@
+with import <nixpkgs> {};
+let
+  certs = pkgs.callPackage ./certs.nix {};
+  pki = pkgs.callPackage ../lib/pki.nix {};
+  cluster = callPackage ../lib/k8s.nix {
+    masterAddress = "10.253.18.100";
+    etcdNodes = [ "etcd0" "etcd1" "etcd2" ];
+    clusterHosts = ''
+          10.253.18.100 k0-0 etcd0 kubernetes
+          10.253.18.101 k0-1 etcd1
+          10.253.18.102 k0-2 etcd2
+          10.253.18.103 k0-3
+          10.253.18.107 k0-4
+          10.253.18.108 k0-5
+          10.253.18.106 fs0-0 fs0-0.local
+          10.1.2.164    fs0-1 fs0-1.local
+          10.253.18.100 fs0-2 fs0-2.local
+          10.253.18.100 itp-registry
+          10.253.18.100 nuget.itpartner.no
+          10.253.18.109 k1-0
+    '';
+    certs = {
+      ca = certs.ca;
+      apiserver = pki.toSet certs.apiserver;
+      kube-proxy = pki.toSet certs.kube-proxy;
+      admin = pki.toSet certs.admin;
+      etcd = pki.toSet certs.etcd;
+      k0-0 = pki.toSet certs.k0-0;
+      k0-1 = pki.toSet certs.k0-1;
+      k0-2 = pki.toSet certs.k0-2;
+      k0-3 = pki.toSet certs.k0-3;
+      k0-4 = pki.toSet certs.k0-4;
+      k0-5 = pki.toSet certs.k0-5;
+    };
+  };
+in
+{
+  k0-0 = { ... }:
+    let
+      apiserver = pki.toSet certs.apiserver;
+    in {
+    require = [ (cluster.apiserver "10.253.18.100" "k0-0" "etcd0") ];
+    boot.kernelModules = [
+      "dm_snapshot"
+      "dm_mirror"
+      "dm_thin_pool"
+    ];
+    # services.dnsmasq.enable = true;
+    fileSystems."/data" = {
+      device = "fs0-0:gv0";
+      fsType = "glusterfs";
+    };
+    fileSystems."/var/lib/docker-registry" = {
+      device = "fs0-0:docker-registry";
+      fsType = "nfs4";
+    };
+    services.dockerRegistry = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+      enableDelete = true;
+      enableGarbageCollect = true;
+      extraConfig = {
+        REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver.cert}";
+        REGISTRY_HTTP_TLS_KEY = "${apiserver.key}";
+      };
+    };
+    environment.systemPackages = [ pkgs.lvm2 ];
+    networking.extraHosts = ''
+      10.253.18.100 itp-registry itp-registry.local
+      10.253.18.100 helm-registry helm-registry.local
+      10.253.18.100 nuget.local
+      10.253.18.100 dashboard.k0.local
+      10.253.18.100 gitlab.k0.local
+      10.253.18.100 baywash.k0.local
+    '';
+    systemd.services.gitlab-upgrade = {
+      description = "Upgrade gitlab by zapping pod";
+      startAt = "Sun 09:00:00";
+      script = "kubectl delete pods -n gitlab --all";
+      path = [ pkgs.kubectl ];
+    };
+  };
+
+  k0-1 = cluster.server "10.253.18.101" "k0-1" "etcd1";
+  k0-2 = cluster.server "10.253.18.102" "k0-2" "etcd2";
+  k0-3 = cluster.worker "10.253.18.103" "k0-3";
+  k0-4 = cluster.worker "10.253.18.107" "k0-4";
+  k0-5 = cluster.worker "10.253.18.108" "k0-5";
+}

clusters/kube0/k0-0.nix

@@ -0,0 +1,25 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/46951acf-de3e-44c3-8d9d-4aa97e691630";
+      fsType = "ext4";
+    };
+  fileSystems."/vol/brick0" =
+    { device = "/dev/gfs_vg/brick0";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube0/k0-1.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/169fb935-f9f5-4689-a15f-7c4ef68553d0";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube0/k0-2.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/362221fe-f5a8-4abe-be5f-6f9bb4fc1d74";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube0/k0-3.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/6b3a3c7b-b8dc-4274-a189-21fbe4d8a488";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube0/k0-4.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/241cdf57-a2b5-482d-a522-01725badc859";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube0/k0-5.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/67441b95-19f2-484d-b57b-3f4b2a55f3cc";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube1/default.nix

@@ -0,0 +1,30 @@
+with import <nixpkgs> {};
+let
+  settings = rec {
+    master = "k1-0";
+    workers = [ "k1-1" "k1-2" ];
+    masterAddress = "10.253.18.109";
+    apiserverAddress = "https://${masterAddress}:8443";
+    clusterHosts = ''
+      10.253.18.109 k1-0 kubernetes fs0-2
+      10.253.18.110 k1-1
+      10.253.18.111 k1-2
+      10.253.18.106 fs0-0
+      10.1.2.164    fs0-1
+      10.253.18.100 k0-0
+      10.253.18.100 gitlab.itpartner.no registry.itpartner.no minio.itpartner.no
+    '';
+  };
+  cluster = callPackage ./k8s.nix { inherit settings; };
+in
+{
+  # k1-0 = cluster.host "10.253.18.109" "k1-0";
+  # k1-1 = cluster.host "10.253.18.110" "k1-1";
+  # k1-2 = cluster.host "10.253.18.111" "k1-2";
+  k1-0 = self:
+  {
+    require = [ (cluster.apiserver "10.253.18.109" "k1-0") ];
+  };
+  k1-1 = cluster.worker "10.253.18.110" "k1-1";
+  k1-2 = cluster.worker "10.253.18.111" "k1-2";
+}

clusters/kube1/deploy.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+id=kube1
+
+# if [ $# = 0 ]; then
+#     echo "usage: deploy.sh name ..."
+#     exit 1
+# fi
+
+if [ ! -f ./deployment.nix ]; then
+    echo "error: ./ does not contain a deployment"
+    exit 1
+fi
+
+# mkdir -p $1/gcroots
+
+# echo "--- Securing certifiates"
+# nix-build -o $1/gcroots/certs $1/build.nix
+
+echo "--- Updating deployment"
+nixops modify -d $id ./deployment.nix
+
+echo "--- Deploying $id"
+nixops deploy -d $id --allow-reboot
+

clusters/kube1/k1-0.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/2e7ba83d-014f-4ef5-a1ce-fc9e34ce7b83";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube1/k1-1.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/70b9d730-9cb6-48e2-8e00-fa78c8feefdf";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube1/k1-2.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/83bb471d-1db7-4c0b-b8aa-8111730a1ea9";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/kube1/k8s.nix

@@ -0,0 +1,136 @@
+{ pkgs, lib, settings, ...}:
+with lib;
+let
+  cluster-ca = pkgs.stdenv.mkDerivation {
+      name = "cluster-ca";
+      src = ./ca;
+      buildCommand = ''
+        mkdir -p $out
+        cp $src/* $out
+      '';
+  };
+  nixos-kubernetes-join-nodes = workers:
+    let
+      wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
+    in
+    pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
+      #!/bin/sh
+      set -e
+      token=$(cat /var/lib/cfssl/apitoken.secret)
+      for i in ${wrk}; do
+         ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
+      done
+    '';
+  cidr = "10.10.0.0/16";
+in
+rec {
+  kubeMaster = {
+    services.cfssl.ca = "${cluster-ca}/ca.pem";
+    services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
+    services.kubernetes = {
+      roles = [ "master" ];
+      masterAddress = settings.master;
+      apiserverAddress = settings.apiserverAddress;
+      clusterCidr = cidr;
+      kubelet.unschedulable = false;
+      pki.genCfsslCACert = false;
+      pki.caCertPathPrefix = "${cluster-ca}/ca";
+      apiserver = {
+        advertiseAddress = settings.masterAddress;
+        authorizationMode = [ "Node" "RBAC" ];
+        securePort = 8443;
+        insecurePort = 8080;
+        extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
+      };
+      addons = {
+        dns = {
+          enable = true;
+          # clusterDomain = "local";
+          reconcileMode = "EnsureExists";
+        };
+      };
+    };
+    networking.firewall = {
+      allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
+      allowedUDPPorts = [ 53 4053 ];
+    };
+    environment.systemPackages = [
+      pkgs.kubernetes-helm
+      (nixos-kubernetes-join-nodes settings.workers)
+    ];
+  };
+
+  kubeWorker = {
+    services.kubernetes = rec {
+      roles = [ "node" ];
+      clusterCidr = cidr;
+      masterAddress = settings.master;
+      apiserverAddress = settings.apiserverAddress;
+    };
+    networking = {
+      firewall = {
+        enable = true;
+        allowedTCPPorts = [ 4194 10250 ];
+        allowedUDPPorts = [ 53 ];
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
+      };
+    };
+    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
+    virtualisation.docker.autoPrune.enable = true;
+  };
+
+  baseNixos = name: {
+    imports = [
+      (../nixos/hardware-configuration + "/${name}.nix")
+      ../nixos/configuration.nix
+    ];
+    security.pki.certificateFiles = [
+      "${cluster-ca}/ca.pem"
+    ];
+    # services.glusterfs = {
+    #   enable = true;
+    #   # tlsSettings = {
+    #     # caCert = certs.ca.caFile;
+    #     # tlsKeyPath = certs.self.keyFile;
+    #     # tlsPem = certs.self.certFile;
+    #   };
+    # };
+    networking = {
+      hostName = name;
+      extraHosts = settings.clusterHosts;
+      # nameservers = [ masterAddress ];
+      # dhcpcd.extraConfig = ''
+      #  static domain_name_servers=${masterAddress}
+      # '';
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 80 443 111 ];
+      firewall.allowedUDPPorts = [ 111 24007 24008 ];
+    };
+  };
+
+  apiserver = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+        kubeMaster
+      ];
+    };
+
+  worker = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+        kubeWorker
+      ];
+    };
+
+  host = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+      ];
+    };
+}