diff --git a/base/configuration.nix b/base/configuration.nix new file mode 100644 index 0000000..567d0f6 --- /dev/null +++ b/base/configuration.nix @@ -0,0 +1,40 @@ +{ config, pkgs, ... }: +{ + # Use the GRUB 2 boot loader. + boot.loader.grub.enable = true; + boot.loader.grub.version = 2; + boot.loader.grub.device = "/dev/sda"; + boot.kernel.sysctl."vm.overcommit_memory"= 1; + + services.vmwareGuest.enable = true; + + # Select internationalisation properties. + i18n = { + consoleFont = "Lat2-Terminus16"; + consoleKeyMap = "us"; + defaultLocale = "en_US.UTF-8"; + }; + + # Set your time zone. + time.timeZone = "Europe/Oslo"; + + networking.search = [ "itpartner.intern" "itpartner.no" ]; + # Enable the OpenSSH daemon. + services.openssh.enable = true; + + # The NixOS release to be compatible with for stateful data such as databases. + system.stateVersion = "17.03"; + programs.zsh.enable = true; + programs.tmux.enable = true; + + virtualisation.docker.enable = true; + + security.rtkit.enable = true; + + fileSystems."/data" = { + device = "10.253.18.103:/data"; + fsType = "nfs"; + }; + + imports = [ ./users.nix ./packages.nix ]; +} diff --git a/share/nix-home.nix b/base/nix-home.nix similarity index 100% rename from share/nix-home.nix rename to base/nix-home.nix diff --git a/base/packages.nix b/base/packages.nix new file mode 100644 index 0000000..055d080 --- /dev/null +++ b/base/packages.nix @@ -0,0 +1,64 @@ +{ config, pkgs, ... }: +let + nix-home = pkgs.callPackage ./nix-home.nix {}; +in + { + nixpkgs.config.allowUnfree = true; + + environment.systemPackages = + with pkgs; + let + sys = [ + stdenv + findutils + coreutils + psmisc + iputils + nettools + netcat + rsync + htop + iotop + zsh + wget + vimNox + python + file + bc + sshuttle + nix-prefetch-git + docker + nix-home + ]; + devel = [ + git + patchelf + binutils + gcc + ]; + dotnet = with dotnetPackages; [ + fsharp + mono-addins + mono + Fake + Paket + ]; + node = with nodePackages; [ + nodejs + npm + npm2nix + yo + gulp + ]; + k8s = [ + ebtables + ethtool + socat + ]; + in + devel ++ + dotnet ++ + node ++ + k8s ++ + sys; +} diff --git a/share/users.nix b/base/users.nix similarity index 88% rename from share/users.nix rename to base/users.nix index ed601ad..eb991c4 100644 --- a/share/users.nix +++ b/base/users.nix @@ -26,4 +26,17 @@ "ssh-dss AAAAB3NzaC1kc3MAAACBANmiPELldjlhW4SKi9NVVN8DIpRouRj2j/v0ycySHYJv0lCE2ATggXMWY/T25eqMTEtwT7U+7g0MIHxR+GLFLpp6N7CiKh3pS0nj4Ig+f9hX2PF5HR6fOgQVCWcNrQTOV6jjZqWjbZpKYDpVfwHZxWrhgAv+9I+w0MHICulDca5RAAAAFQCjmmSZGG137bgGuLPXZkyvc8DXswAAAIB1bMTmPWS1qsZ5H4hgzoKcW+5b+yD7Yn62GFmZS/n4RdyJt7gBJwxukXaTs9B5g922lem4Tk6W6kslCzAu6Y7JDOkhX/hWasb6fGdCmmK/btqwi2imGeVJImAoFoTKfm4JprKcOmSATGMgTlzFHYFDpngyZ9pFnyubI829zfzNTwAAAIEAhwQwqEhBOT8cEKZRiDExi7jBk7zRKYhX1Wb6uUKI07qQFTLehUIahirHxqXcDhlcxzgHcwXKt6CBPYvre9qhqP6865Be/KecYycntVsx/o77Hv5bqETXojhLhb8I3hD1UnxIJ1FXVOnhL+SbO46oICsghBApDtgTX9iRFnuw0fU= jonas" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvIIQoi0aM2XmX7evnte/sQTCczSYYg0O0KT7g6Xao4nAoiMZ7udxOijd0vD8VBRSVuz6epcuPLsD0z6skiCJFpT0gG0KkTS6dLZMD3+KOvmvolpuGvRv6Ad/bO05YUvGHJAPdwwCxcXajtBoHOd+KUq8xBqyexgi20i+4P/JulY+RQKPlnQHlb6glcDAjt9RPh96t9T5lCoMAqMWtzV9GZE8/H+o6nMf9pxTjxT/oW/8EKZVgDgCSnpZg668Xj0UNcJW/ba3kSpjUsrdvZgM1E3TVgJ/YZDpM01m9hHS7PBcsJu6RyUMhamAVlsYS4Vy2ylU5rYAPTh23CZh24KD3+AtzTd9vhDNLz+KNKDzgW5b3IHoOyXG78RtPw7gHBGlGQc9OlJqTSLZAFWE2PNP1Pa6q0mqWcJuakSwyQoHvS+8PgEBr0eKKebwuXauuft1DvMNpi9SFxJY5Oy7ck62WzbBDNFDOTdNabXUV/QdOv5Zc2XrUCkr6C1dk57C0G8KTJCkqZIV1XMYZIlDZPuqlK8jZdPjRAnRmkYvvitb0FsvixpsWll1PRj9bYYZ01xX6djuDRvc1FeRG199HKOVWN53l9EMTpy0coxl2hZpbnyfxLdS/tkPkoSuWeY/MKWrn4yd1Q2+edjhTvnn6o6kfnKXKBkuoQvdaX/nyig+X+Q== august.s.solvang@gmail.com" ]; + + security.sudo.wheelNeedsPassword = false; + security.sudo.extraConfig = + '' + Defaults:root,%wheel env_keep+=LOCALE_ARCHIVE + Defaults:root,%wheel env_keep+=NIX_PATH + Defaults:root,%wheel env_keep+=TERMINFO_DIRS + Defaults env_keep+=SSH_AUTH_SOCK + Defaults lecture=never + Defaults shell_noargs + root ALL=(ALL) SETENV: ALL + %wheel ALL=(ALL) NOPASSWD: ALL, SETENV: ALL + ''; } diff --git a/share/workstation.nix b/base/workstation.nix similarity index 100% rename from share/workstation.nix rename to base/workstation.nix diff --git a/git.nix b/git.nix index 838b5d8..0557631 100644 --- a/git.nix +++ b/git.nix @@ -1,7 +1,19 @@ { git01 = { config, lib, pkgs, ... }: - { - deployment.targetHost = "10.253.18.103"; - imports = [ ./git01/configuration.nix ]; + { + deployment.targetHost = "10.253.18.103"; + networking.hostName = "git01"; # Define your hostname + imports = [ ./hw/git01.nix ./git01/configuration.nix ]; + services.nfs.server = { + enable=true; + exports= '' + /data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) + /data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) + /data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) + /data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) + ''; }; + networking.firewall.allowedTCPPorts = [2049 111 20048]; + networking.firewall.allowedUDPPorts = [2049 111 20048]; + }; } diff --git a/git01/configuration.nix b/git01/configuration.nix deleted file mode 100644 index 7d85681..0000000 --- a/git01/configuration.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ config, pkgs, ... }: -{ - imports = [ - ./hardware-configuration.nix - ../share/configuration.nix - ../share/users.nix - ]; - - networking.hostName = "git01"; # Define your hostname - networking.firewall.allowedTCPPorts = [2049 111 20048]; - networking.firewall.allowedUDPPorts = [2049 111 20048]; - boot.loader.grub.device = "/dev/sda"; - - services.nfs.server = { - enable=true; - - exports= '' - /data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) - /data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) - /data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) - /data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0) - ''; - }; -} diff --git a/git01/hardware-configuration.nix b/hw/git01.nix similarity index 100% rename from git01/hardware-configuration.nix rename to hw/git01.nix diff --git a/k8s0-0/hardware-configuration.nix b/hw/k8s0-0.nix similarity index 100% rename from k8s0-0/hardware-configuration.nix rename to hw/k8s0-0.nix diff --git a/k8s0-1/hardware-configuration.nix b/hw/k8s0-1.nix similarity index 100% rename from k8s0-1/hardware-configuration.nix rename to hw/k8s0-1.nix diff --git a/k8s0-2/hardware-configuration.nix b/hw/k8s0-2.nix similarity index 100% rename from k8s0-2/hardware-configuration.nix rename to hw/k8s0-2.nix diff --git a/k8s.nix b/k8s.nix index 318e070..c23ac87 100644 --- a/k8s.nix +++ b/k8s.nix @@ -1,19 +1,129 @@ -{ - k8s00 = { config, lib, pkgs, ... }: - { - deployment.targetHost = "10.253.18.101"; - imports = [ ./k8s00/configuration.nix ]; +let + etcdConfig = name: { + services.etcd = { + inherit name; + advertiseClientUrls = [ "https://${name}:2379" ]; + initialAdvertisePeerUrls = [ "https://${name}:2380" ]; + enable = true; + certFile = ./pki/etcd.pem; + keyFile = ./pki/etcd-key.pem; + trustedCaFile = ./pki/ca.pem; + peerClientCertAuth = true; + listenClientUrls = ["https://0.0.0.0:2379"]; + listenPeerUrls = ["https://0.0.0.0:2380"]; + initialCluster = [ + "etcd0=https://etcd0:2380" + "etcd1=https://etcd1:2380" + ]; + # environment.variables = { + # ETCDCTL_CERT_FILE = ./pki/client.pem; + # ETCDCTL_KEY_FILE = ./pki/client-key.pem; + # ETCDCTL_CA_FILE = ./pki/ca.pem; + # ETCDCTL_PEERS = "https://127.0.0.1:2379"; + # }; + }; + networking.firewall.allowedTCPPorts = [ 2379 2380 ]; + }; + + flannelConfig = { + services.flannel = { + enable = true; + network = "10.10.0.0/16"; + iface = "enp0s3"; + etcd = { + endpoints = ["https://etcd0:2379" "https://etcd1:2379" ]; + certFile = ./pki/client.pem; + keyFile = ./pki/client-key.pem; + caFile = ./pki/ca.pem; + }; + }; + }; + + kubeNode = { + services.kubernetes = { + # verbose = true; + roles = [ "node" ]; + kubeconfig = { + server = "https://kubernetes:443"; + caFile = ./pki/ca.pem; + certFile = ./pki/client.pem; + keyFile = ./pki/client-key.pem; + }; + etcd = { + servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; + certFile = ./pki/client.pem; + keyFile = ./pki/client-key.pem; + caFile = ./pki/ca.pem; + }; + # kubelet.clusterDns = "10.10.1.1"; }; - k8s01 = { config, lib, pkgs, ... }: + networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN + networking.firewall.allowedTCPPorts = [ 10250 ]; + networking.extraHosts = '' + 10.253.18.100 etcd0 kubernetes + 10.253.18.101 etcd1 + ''; + + systemd.services.docker.after = [ "flannel.service" ]; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; + virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; + }; + + kubeMaster = { + services.dockerRegistry = { + enable = true; + listenAddress = "0.0.0.0"; + }; + + services.kubernetes = { + roles = [ "master" ]; + apiserver = { + publicAddress = "0.0.0.0"; + address = "0.0.0.0"; + clientCaFile = ./pki/ca.pem; + tlsCertFile = ./pki/apiserver.pem; + tlsKeyFile = ./pki/apiserver-key.pem; + kubeletClientCaFile = ./pki/ca.pem; + kubeletClientCertFile = ./pki/client.pem; + kubeletClientKeyFile = ./pki/client-key.pem; + }; + scheduler.leaderElect = true; + controllerManager.leaderElect = true; + controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem; + }; + + networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; + networking.firewall.allowedUDPPorts = [ 53 ]; + systemd.services.flannel.after = [ "etcd.service" ]; + }; +in +{ + k8s0-0 = { config, lib, pkgs, ... }: + let etcd = etcdConfig "etcd0"; in { deployment.targetHost = "10.253.18.100"; - imports = [ ./k8s01/configuration.nix ]; + networking.hostName = "k8s0-0"; + imports = [ ./hw/k8s0-0.nix ./base/configuration.nix ]; + require = [ etcd flannelConfig ]; + # require = [ etcd flannelConfig kubeMaster kubeNode ]; }; - k8s02 = { config, lib, pkgs, ... }: + k8s0-1 = { config, lib, pkgs, ... }: + let etcd = etcdConfig "etcd1"; in + { + deployment.targetHost = "10.253.18.101"; + networking.hostName = "k8s0-1"; + imports = [ ./hw/k8s0-1.nix ./base/configuration.nix ]; + require = [ etcd flannelConfig ]; + # require = [ etcd flannelConfig kubeNode ]; + }; + + k8s0-2 = { config, lib, pkgs, ... }: { deployment.targetHost = "10.253.18.102"; - imports = [ ./k8s02/configuration.nix ]; - }; + networking.hostName = "k8s0-2"; + imports = [ ./hw/k8s0-2.nix ./base/configuration.nix ]; + require = [ flannelConfig ]; + }; } diff --git a/k8s0-0/configuration.nix b/k8s0-0/configuration.nix deleted file mode 100644 index 43efb44..0000000 --- a/k8s0-0/configuration.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ config, pkgs, ... }: -{ - imports = [ - ./hardware-configuration.nix - ../share/configuration.nix - ../share/users.nix - ../share/k8s.nix - ]; - - networking.hostName = "kubemaster"; # Define your hostname. - networking.firewall.allowedTCPPorts = [ 5000 8080 443 ]; - - boot.loader.grub.device = "/dev/sda"; - - services.dockerRegistry = { - enable = true; - listenAddress = "0.0.0.0"; - }; - - services.kubernetes.roles = [ "master" ]; - services.kubernetes.apiserver = { - publicAddress = "0.0.0.0"; - address = "0.0.0.0"; - # kubeletClientCaFile = ../pki/ca.pem; - # kubeletClientCertFile = ../pki/client.pem; - # kubeletClientKeyFile = ../pki/client-key.pem; - clientCaFile = ../pki/ca.pem; - tlsCertFile = ../pki/apiserver.pem; - tlsKeyFile = ../pki/apiserver-key.pem; - }; - services.kubernetes.kubeconfig = { - certFile = ../pki/kubemaster.pem; - keyFile = ../pki/kubemaster-key.pem; - }; -} diff --git a/k8s0-1/configuration.nix b/k8s0-1/configuration.nix deleted file mode 100644 index ecd1574..0000000 --- a/k8s0-1/configuration.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ config, pkgs, ... }: -{ - imports = [ - ./hardware-configuration.nix - ../share/configuration.nix - ../share/users.nix - ../share/k8s.nix - ]; - - boot.loader.grub.device = "/dev/sda"; - - networking.hostName = "kub01"; # Define your hostname. - - services.kubernetes.kubeconfig = { - certFile = ../pki/kub01.pem; - keyFile = ../pki/kub01-key.pem; - }; -} diff --git a/k8s0-2/configuration.nix b/k8s0-2/configuration.nix deleted file mode 100644 index d0684eb..0000000 --- a/k8s0-2/configuration.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ config, pkgs, ... }: -{ - imports = [ - ./hardware-configuration.nix - ../share/configuration.nix - ../share/users.nix - ../share/k8s.nix - ]; - boot.loader.grub.device = "/dev/sda"; - - networking.hostName = "kub02"; # Define your hostname. - - services.kubernetes.kubeconfig = { - certFile = ../pki/kub02.pem; - keyFile = ../pki/kub02-key.pem; - }; -} diff --git a/pki/apiserver.json b/pki/apiserver.json index 3d355a6..d1da0e7 100644 --- a/pki/apiserver.json +++ b/pki/apiserver.json @@ -1,7 +1,7 @@ { "hosts": [ - "kubemaster", - "10.253.18.101" + "k8s0-0", + "10.253.18.100" ], "key": { "algo": "rsa", diff --git a/pki/ca.json b/pki/ca.json index 21e249b..e7ebfa1 100644 --- a/pki/ca.json +++ b/pki/ca.json @@ -1,5 +1,6 @@ { "hosts": [ + "itpartner.no", "itpartner.intern", "cluster.local" ], diff --git a/pki/client.json b/pki/client.json index c4fb879..5a28665 100644 --- a/pki/client.json +++ b/pki/client.json @@ -1,9 +1,4 @@ { - "CN": "@host@", - "hosts": [ - "@host@", - "@ip@" - ], "key": { "algo": "rsa", "size": 2048 diff --git a/pki/etcd.json b/pki/etcd.json new file mode 100644 index 0000000..2d3cf02 --- /dev/null +++ b/pki/etcd.json @@ -0,0 +1,19 @@ +{ + "hosts": [ + "etcd0", + "etcd1" + ], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "NO", + "L": "Tromsø", + "O": "Serit IT Partner Tromsø AS", + "OU": "", + "ST": "" + } + ] +} diff --git a/pki/mkcerts.sh b/pki/mkcerts.sh index eb8d4f3..8b66987 100755 --- a/pki/mkcerts.sh +++ b/pki/mkcerts.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -hosts="kubemaster,101 kub01,100 kub02,102" +# hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102" mkcacert () { cfssl genkey -initca ca.json | cfssljson -bare ca @@ -11,37 +11,51 @@ mkapicert () { | cfssljson -bare apiserver } +mketcdcert () { + cfssl gencert -ca ca.pem -ca-key ca-key.pem etcd.json \ + | cfssljson -bare etcd +} + mkclientcert () { - host=$1 - ip=$2 - - sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \ - | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \ - | cfssljson -bare $host + cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \ + | cfssljson -bare client } -mkclientcerts () { - for i in $hosts; do - IFS="," - set -- $i - mkclientcert $1 10.253.18.$2 - done -} +# mkclientcert () { +# host=$1 +# ip=$2 + +# sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \ +# | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \ +# | cfssljson -bare $host +# } + +# mkclientcerts () { +# for i in $hosts; do +# IFS="," +# set -- $i +# mkclientcert $1 10.253.18.$2 +# done +# } case $1 in all) mkcacert mkapicert - mkclientcerts + mketcdcert + mkclientcert ;; - clients) - mkclientcerts + client) + mkclientcert ;; api) mkapicert ;; + etcd) + mketcdcert + ;; *) - echo "usege: mkcerts.sh (all|clients|api)" + echo "usege: mkcerts.sh (all|client|api|etcd)" exit 1 ;; esac diff --git a/share/configuration.nix b/share/configuration.nix deleted file mode 100644 index e7d3e9a..0000000 --- a/share/configuration.nix +++ /dev/null @@ -1,112 +0,0 @@ -{ config, pkgs, ... }: -let - nix-home = pkgs.callPackage ./nix-home.nix {}; -in -{ - # Use the GRUB 2 boot loader. - boot.loader.grub.enable = true; - boot.loader.grub.version = 2; - boot.kernel.sysctl."vm.overcommit_memory"= 1; - - services.vmwareGuest.enable=true; - - # Select internationalisation properties. - i18n = { - consoleFont = "Lat2-Terminus16"; - consoleKeyMap = "us"; - defaultLocale = "en_US.UTF-8"; - }; - - # Set your time zone. - time.timeZone = "Europe/Oslo"; - - networking.search = [ "itpartner.intern" ]; - # Enable the OpenSSH daemon. - services.openssh.enable = true; - - # The NixOS release to be compatible with for stateful data such as databases. - system.stateVersion = "17.09"; - - programs.zsh.enable = true; - programs.tmux.enable = true; - - virtualisation.docker.enable = true; - - security.sudo.wheelNeedsPassword = false; - security.sudo.extraConfig = - '' - Defaults:root,%wheel env_keep+=LOCALE_ARCHIVE - Defaults:root,%wheel env_keep+=NIX_PATH - Defaults:root,%wheel env_keep+=TERMINFO_DIRS - Defaults env_keep+=SSH_AUTH_SOCK - Defaults lecture=never - Defaults shell_noargs - root ALL=(ALL) SETENV: ALL - %wheel ALL=(ALL) NOPASSWD: ALL, SETENV: ALL - ''; - - security.rtkit.enable = true; - - fileSystems."/data" = { - device = "10.253.18.103:/data"; - fsType = "nfs"; - }; - - nixpkgs.config.allowUnfree = true; - environment.systemPackages = - with pkgs; - let - sys = [ - stdenv - findutils - coreutils - psmisc - iputils - nettools - netcat - rsync - htop - iotop - zsh - wget - vimNox - python - file - bc - sshuttle - nix-prefetch-git - docker - nix-home - ]; - devel = [ - git - patchelf - binutils - gcc - ]; - dotnet = with dotnetPackages; [ - fsharp - mono-addins - mono - Fake - Paket - ]; - node = with nodePackages; [ - nodejs - npm - npm2nix - yo - gulp - ]; - k8s = [ - ebtables - ethtool - socat - ]; - in - devel ++ - dotnet ++ - node ++ - k8s ++ - sys; -} diff --git a/share/k8s-node.nix b/share/k8s-node.nix deleted file mode 100644 index b5d8893..0000000 --- a/share/k8s-node.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ config, pkgs, ... }: -{ - services.kubernetes.roles = [ "node" ]; - services.kubernetes.kubeconfig = { - server = "https://10.253.18.101:443"; - # caFile = ../pki/ca.cert.pem; - # certFile = ../pki/certificate.pem; - # keyFile = ../pki/key.pem; - }; - services.kubernetes.apiserver = { - # kubeletClientCaFile = ../pki/ca.cert.pem; - # kubeletClientCertFile = ../pki/certificate.pem; - # kubeletClientKeyFile = ../pki/key.pem; - # clientCaFile = ../pki/ca.cert.pem; - tlsCertFile = ../pki/certificate.pem; - tlsKeyFile = ../pki/key.pem; - }; - services.kubernetes.kubelet = { - tlsCertFile = ../pki/certificate.pem; - tlsKeyFile = ../pki/key.pem; - }; -} diff --git a/share/k8s.nix b/share/k8s.nix deleted file mode 100644 index bb1aa92..0000000 --- a/share/k8s.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ config, pkgs, ... }: -{ - services.kubernetes.roles = [ "node" ]; - services.kubernetes.kubeconfig = { - server = "https://10.253.18.101:443"; - caFile = ../pki/ca.pem; - }; - networking.firewall.allowedTCPPorts = [ 10250 ]; -}