Works!

2017-08-18 11:31:07 +02:00
parent e2c2c6b811
commit 85a045577d
13 changed files with 458 additions and 236 deletions
--- a/base/configuration.nix
+++ b/base/configuration.nix
@@ -16,7 +16,7 @@
  # Set your time zone.
  time.timeZone = "Europe/Oslo";

-  networking.search = [ "itpartner.intern" "itpartner.no" ];
+  networking.search = [ "itpartner.no" "itpartner.intern" ];

  services.openssh.enable = true;
  services.nfs.server.enable = true;
--- a/base/pki.nix
+++ b/base/pki.nix
@@ -11,26 +11,17 @@ let
                  "expiry": "43800h",
                  "usages": [
                      "signing",
-                  "key encipherment",
-                  "server auth"
-                      ]
+                      "key encipherment",
+                      "server auth"
+                  ]
              },
              "client": {
                  "expiry": "43800h",
                  "usages": [
                      "signing",
-                  "key encipherment",
-                  "client auth"
-                      ]
-              },
-              "peer": {
-                  "expiry": "43800h",
-                  "usages": [
-                      "signing",
-                  "key encipherment",
-                  "server auth",
-                  "client auth"
-                      ]
+                      "key encipherment",
+                      "client auth"
+                  ]
              }
          }
      }
@@ -82,4 +73,52 @@ in
      buildInputs = [ pkgs.cfssl ];
    } (cfssl cert.csr cert.profile);

+  # server-cert = mkCert {
+  #     name = "kubernetes";
+  #     csr = csr {
+  #       cn = "kubernetes";
+  #       hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
+  #     };
+  #     profile = "server";
+  # };
+
+  # etcd0-cert = mkCert {
+  #   name = "etcd0";
+  #   csr = csr {
+  #     cn = "etcd0";
+  #     hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
+  #   };
+  #   profile = "peer";
+  # };
+
+  # etcd1-cert = mkCert {
+  #   name = "etcd1";
+  #   csr = csr {
+  #     cn = "etcd1";
+  #     hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
+  #   };
+  #   profile = "peer";
+  # };
+
+  # client-cert = mkCert {
+  #   name = "client";
+  #   csr = csr {
+  #     cn = "client";
+  #     hosts = '''';
+  #   };
+  #   profile = "client";
+  # };
+
+  # server_key = "${server-cert}/cert-key.pem";
+  # server_cert = "${server-cert}/cert.pem";
+
+  # etcd0_key = "${etcd0-cert}/cert-key.pem";
+  # etcd0_cert = "${etcd0-cert}/cert.pem";
+
+  # etcd1_key = "${etcd1-cert}/cert-key.pem";
+  # etcd1_cert = "${etcd1-cert}/cert.pem";
+
+  # client_key = "${client-cert}/cert-key.pem";
+  # client_cert = "${client-cert}/cert.pem";
+
  }
--- a/bin/deploy.sh
+++ b/bin/deploy.sh
--- a/bin/teardown.sh
+++ b/bin/teardown.sh
@@ -8,8 +8,9 @@ fi
 d=$1
 f=.$d.$$

-nixops ssh -d $d ${d}0-0 kubectl delete --all pods
-nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
+# nixops ssh -d $d ${d}0-0 kubectl delete --all pods
+# nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
+# sleep 60

 sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f
 nixops modify -d $d $f
@@ -17,4 +18,5 @@ nixops deploy -d $d
 rm $f

 nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"
+nixops ssh-for-each -d $d reboot

--- a/certs.nix
+++ b/certs.nix
@@ -0,0 +1,149 @@
+let
+  pkgs = import <nixpkgs> {};
+
+  runWithOpenSSL = file: cmd: pkgs.runCommand file {
+    buildInputs = [ pkgs.openssl ];
+  } cmd;
+
+  etcd_cnf = pkgs.writeText "etcd-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = digitalSignature, keyEncipherment
+    extendedKeyUsage = serverAuth
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = etcd0
+    DNS.2 = etcd1
+    DNS.3 = etcd2
+    DNS.4 = k8s0-0
+    DNS.5 = k8s0-1
+    DNS.6 = k8s0-2
+    IP.1 = 127.0.0.1
+  '';
+
+  etcd_client_cnf = pkgs.writeText "etcd-client-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = digitalSignature, keyEncipherment
+    extendedKeyUsage = clientAuth
+  '';
+
+  apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = kubernetes
+    DNS.2 = kubernetes.default
+    DNS.3 = kubernetes.default.svc
+    DNS.4 = kubernetes.default.svc.cluster.local
+    IP.1 = 10.10.10.1
+    IP.2 = 10.253.18.100
+  '';
+
+  worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = k8s0-0
+    DNS.2 = k8s0-1
+    DNS.3 = k8s0-2
+  '';
+
+  ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
+  ca_pem = runWithOpenSSL "ca.pem" ''
+    openssl req \
+      -x509 -new -nodes -key ${ca_key} \
+      -days 10000 -out $out -subj "/CN=etcd-ca"
+  '';
+
+  etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
+  etcd_csr = runWithOpenSSL "etcd.csr" ''
+    openssl req \
+      -new -key ${etcd_key} \
+      -out $out -subj "/CN=etcd" \
+      -config ${etcd_cnf}
+  '';
+  etcd_cert = runWithOpenSSL "etcd.pem" ''
+    openssl x509 \
+      -req -in ${etcd_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} \
+      -CAcreateserial -out $out \
+      -days 365 -extensions v3_req \
+      -extfile ${etcd_cnf}
+  '';
+
+  etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
+    "openssl genrsa -out $out 2048";
+  etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''
+    openssl req \
+      -new -key ${etcd_client_key} \
+      -out $out -subj "/CN=etcd-client" \
+      -config ${etcd_client_cnf}
+  '';
+  etcd_client_cert = runWithOpenSSL "etcd-client.pem" ''
+    openssl x509 \
+      -req -in ${etcd_client_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${etcd_client_cnf}
+  '';
+
+  apiserver_key = runWithOpenSSL "apiserver-key.pem"
+    "openssl genrsa -out $out 2048";
+  apiserver_csr = runWithOpenSSL "apiserver.csr" ''
+    openssl req \
+      -new -key ${apiserver_key} \
+      -out $out -subj "/CN=kube-apiserver" \
+      -config ${apiserver_cnf}
+  '';
+  apiserver_cert = runWithOpenSSL "apiserver.pem" ''
+    openssl x509 \
+      -req -in ${apiserver_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${apiserver_cnf}
+  '';
+
+  worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
+  worker_csr = runWithOpenSSL "worker.csr" ''
+    openssl req \
+      -new -key ${worker_key} \
+      -out $out -subj "/CN=kube-worker" \
+      -config ${worker_cnf}
+  '';
+  worker_cert = runWithOpenSSL "worker.pem" ''
+    openssl x509 \
+      -req -in ${worker_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${worker_cnf}
+  '';
+
+in
+{
+  inherit ca_key ca_pem;
+  inherit etcd_key etcd_cert;
+  inherit etcd_client_key etcd_client_cert;
+  inherit apiserver_key apiserver_cert;
+  inherit worker_key worker_cert;
+}
--- a/git.nix
+++ b/git.nix
@@ -1,19 +0,0 @@
-{
-  git01 = { config, lib, pkgs, ... }:
-  {
-    deployment.targetHost = "10.253.18.103";
-    networking.hostName = "git01"; # Define your hostname
-    imports = [ ./hw/git01.nix ./git01/configuration.nix ];
-    services.nfs.server = {
-      enable=true;
-      exports= ''
-        /data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-        /data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-        /data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-        /data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-      '';
-    };
-    networking.firewall.allowedTCPPorts = [2049 111 20048];
-    networking.firewall.allowedUDPPorts = [2049 111 20048];
-  };
-}
--- a/k8s.nix
+++ b/k8s.nix
@@ -1,107 +1,57 @@
-with import ./base/pki.nix;
+with import ./certs.nix;
 let
-  server-cert = mkCert {
-      name = "kubernetes";
-      csr = csr {
-        cn = "kubernetes";
-        hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
-      };
-      profile = "server";
-  };
+  etcdServers = [ "etcd0" "etcd1" "etcd2" ];
+  # etcdServers = [ "k8s0-0" "k8s0-1" "k8s0-2" ];
+  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
+  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdServers;

-  etcd0-cert = mkCert {
-    name = "etcd0";
-    csr = csr {
-      cn = "etcd0";
-      hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
-    };
-    profile = "peer";
-  };
-
-  etcd1-cert = mkCert {
-    name = "etcd1";
-    csr = csr {
-      cn = "etcd1";
-      hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
-    };
-    profile = "peer";
-  };
-
-  client-cert = mkCert {
-    name = "client";
-    csr = csr {
-      cn = "client";
-      hosts = '''';
-    };
-    profile = "client";
-  };
-
-  server_key = "${server-cert}/cert-key.pem";
-  server_cert = "${server-cert}/cert.pem";
-
-  etcd0_key = "${etcd0-cert}/cert-key.pem";
-  etcd0_cert = "${etcd0-cert}/cert.pem";
-
-  etcd1_key = "${etcd1-cert}/cert-key.pem";
-  etcd1_cert = "${etcd1-cert}/cert.pem";
-
-  client_key = "${client-cert}/cert-key.pem";
-  client_cert = "${client-cert}/cert.pem";
-
-  etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ];
-  etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ];
-
-  etcdConfig = etcd: {
+  etcdConfig = name: {
    services.etcd = {
-      name = etcd.name;
+      inherit name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
-      keyFile  = "${etcd.key}";
-      certFile = "${etcd.cert}";
-      trustedCaFile = "${ca_cert}";
-      advertiseClientUrls = [ "https://${etcd.name}:2379" ];
-      initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ];
+      keyFile  = etcd_key;
+      certFile = etcd_cert;
+      trustedCaFile = ca_pem;
+      advertiseClientUrls = [ "https://${name}:2379" ];
+      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
-      ETCDCTL_KEY_FILE  = "${etcd.key}";
-      ETCDCTL_CERT_FILE = "${etcd.cert}";
-      ETCDCTL_CA_FILE   = "${ca_cert}";
-      ETCDCTL_PEERS     = "https://localhost:2379";
+      ETCDCTL_KEY_FILE  = "${etcd_client_key}";
+      ETCDCTL_CERT_FILE = "${etcd_client_cert}";
+      ETCDCTL_CA_FILE   = "${ca_pem}";
+      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
    };
-    # networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

-  flannelConfig = {
+  kubeConfig = {
    services.flannel = {
      enable = true;
      network = "10.10.0.0/16";
-      iface = "enp2s0";
+      iface = "ens32";
      etcd = {
-        endpoints = etcdServers;
-        caFile   = "${ca_cert}";
-        keyFile  = "${client_key}";
-        certFile = "${client_cert}";
+        endpoints = etcdEndpoints;
+        keyFile  = etcd_client_key;
+        certFile = etcd_client_cert;
+        caFile   = ca_pem;
      };
    };
-    # networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
-  };
-
-  kubeConfig = {
+    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
    systemd.services.docker = {
      after = [ "flannel.service" ];
      serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
    };
-    virtualisation.docker.extraOptions =
-      "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
+    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU";
    services.kubernetes.etcd = {
-      servers = etcdServers;
-      caFile   = "${ca_cert}";
-      keyFile  = "${client_key}";
-      certFile = "${client_cert}";
+      servers = etcdEndpoints;
+      keyFile  = etcd_client_key;
+      certFile = etcd_client_cert;
+      caFile   = ca_pem;
    };
    # services.kubernetes.verbose = true;
  };
@@ -110,26 +60,26 @@ let
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
-        server = "https://10.253.18.100:443";
-        caFile   = "${ca_cert}";
-        keyFile  = "${client_key}";
-        certFile = "${client_cert}";
+        server = "https://10.253.18.100:4443";
+        keyFile  = worker_key;
+        certFile = worker_cert;
+        caFile   = ca_pem;
      };
      kubelet = {
-        tlsKeyFile  = "${client_key}";
-        tlsCertFile = "${client_cert}";
+        tlsKeyFile  = worker_key;
+        tlsCertFile = worker_cert;
        networkPlugin = null;
-        clusterDns = "10.10.21.0";
+        clusterDns = "10.253.18.100";
      };
    };
-    networking.firewall = {
-      enable = false;
-      # allowedTCPPorts = [ 53 10250 8000 8080 ];
-      # allowedUDPPorts = [ 53 ];
-      # trustedInterfaces = [ "flannel.1" "docker0" ];
-      # extraCommands = ''
-      #    iptables -P FORWARD ACCEPT
-      # '';
+    networking = {
+      firewall = {
+        enable = true;
+        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
+        allowedTCPPorts = [ 53 10250 ];
+        allowedUDPPorts = [ 53 ];
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
+      };
    };
  };

@@ -137,44 +87,46 @@ let
    services.kubernetes = {
      roles = [ "master" ];
      apiserver = {
-        publicAddress = "0.0.0.0";
        address = "0.0.0.0";
-        clientCaFile = "${ca_cert}";
-        tlsKeyFile   = "${server_key}";
-        tlsCertFile  = "${server_cert}";
-        # serviceAccountKeyFile = "${server_key}";
-        # kubeletClientCaFile   = "${ca_cert}";
-        # kubeletClientKeyFile  = "${client_key}";
-        # kubeletClientCertFile = "${client_cert}";
+        publicAddress = "0.0.0.0";
+        advertiseAddress = "10.253.18.100";
+        securePort = 4443;
+        tlsKeyFile   = apiserver_key;
+        tlsCertFile  = apiserver_cert;
+        clientCaFile = ca_pem;
+        kubeletClientCaFile   = ca_pem;
+        kubeletClientKeyFile  = worker_key;
+        kubeletClientCertFile = worker_cert;
+        # serviceAccountKeyFile = apiserver_key;
      };
      scheduler.leaderElect = true;
      controllerManager.leaderElect = true;
-      controllerManager.serviceAccountKeyFile = "${server_key}";
+      controllerManager.serviceAccountKeyFile = apiserver_key;
+      dns.enable = true;
+      dns.port = 4053;
+    };
+    networking.firewall = {
+      allowedTCPPorts = [ 5000 8080 4443 4053 ];
+      allowedUDPPorts = [ 4053 ];
    };
-    # networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
-    # networking.firewall.allowedUDPPorts = [ 53 ];
  };

  baseConfig = node: {
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
-    networking.hostName = node;
-    networking.extraHosts = ''
-      10.253.18.100 etcd0 k8s0-0 kubernetes
-      10.253.18.101 etcd1
-    '';
-    virtualisation.docker.enable = true;
-  };
-
-  etcdConf0 = etcdConfig {
-    name = "etcd0";
-    key = etcd0_key;
-    cert = etcd0_cert;
-  };
-
-  etcdConf1 = etcdConfig {
-    name = "etcd1";
-    key = etcd1_key;
-    cert = etcd1_cert;
+    networking = {
+      hostName = node;
+      extraHosts = ''
+        10.253.18.100 etcd0 kubernetes
+        10.253.18.101 etcd1
+        10.253.18.102 etcd2
+      '';
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 80 443 ];
+    };
+    services.dnsmasq.enable = true;
+    services.dnsmasq.servers = [
+      "/cluster.local/10.253.18.100#4053"
+    ];
  };

  minion = host: ip: { config, lib, pkgs, ... }:
@@ -184,18 +136,19 @@ let
    in
    {
      deployment.targetHost = ip;
-      require = [ base flannelConfig kubeConfig kubeNode ];
-  };
+      require = [ base kubeConfig kubeNode ];
+      services.kubernetes.dns.enable = false;
+    };
 in
 {
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-0";
-      etcd = etcdConf0;
+      etcd = etcdConfig "etcd0";
    in
    {
      deployment.targetHost = "10.253.18.100";
-      require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ];
+      require = [ base etcd kubeConfig kubeMaster kubeNode ];
      services.dockerRegistry = {
        enable = true;
        listenAddress = "0.0.0.0";
@@ -205,12 +158,22 @@ in
  k8s0-1 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-1";
-      etcd = etcdConf1;
+      etcd = etcdConfig "etcd1";
    in
    {
      deployment.targetHost = "10.253.18.101";
-      require = [ base etcd flannelConfig kubeConfig kubeNode ];
-  };
+      require = [ base etcd kubeConfig kubeNode ];
+      services.kubernetes.dns.enable = false;
+    };

-  k8s0-2 = minion "k8s0-2" "10.253.18.102";
+  k8s0-2 = { config, lib, pkgs, ... }:
+    let
+      base = baseConfig "k8s0-2";
+      etcd = etcdConfig "etcd2";
+    in
+    {
+      deployment.targetHost = "10.253.18.102";
+      require = [ base etcd kubeConfig kubeNode ];
+      services.kubernetes.dns.enable = false;
+    };
 }
--- a/test/gitlab.yaml
+++ b/test/gitlab.yaml
@@ -1,56 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gitlab
-  labels:
-    run: gitlab
-spec:
-  type: NodePort
-  ports:
-  - port: 80
-    targetPort: 80
-    protocol: TCP
-    name: http
-  - port: 443
-    protocol: TCP
-    name: https
-  - port: 22
-    protocol: TCP
-    name: ssh
-  selector:
-   run: gitlab
---
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: gitlab
-  namespace: default
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: busybox
-      containers:
-      - image: gitlab/gitlab-ce:latest
-        name: gitlab-container
-        ports:
-        - containerPort: 80
-        - containerPort: 22
-        - containerPort: 443
-      volumeMounts:
-      - mountPath: /etc/gitlab
-        name: gitlab-data
-        subPath: config
-      - mountPath: /var/log/gitlab
-        name: gitlab-data
-        subPath: logs
-      - mountPath: /var/opt/gitlab
-        name: gitlab-data
-        subPath: data
-      volumes:
-      - name: gitlab-data
-        nfs:
-          server: 10.253.10.103
-          path: /data/gitlab
-          ReadOnly: false
--- a/test/busybox.yaml
+++ b/test/busybox.yaml
@@ -5,17 +5,14 @@ metadata:
  labels:
    run: bbox
 spec:
-  type: NodePort
+  type: ClusterIP
+  selector:
+   app: busybox
  ports:
-  - port: 8080
-    targetPort: 80
+  - port: 8000
+    targetPort: 8000
    protocol: TCP
    name: http
-  - port: 443
-    protocol: TCP
-    name: https
-  selector:
-   run: bbox
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -33,13 +30,15 @@ spec:
      - image: busybox
        name: busybox
        command:
-          - sleep
-          - "3600"
-        ports:
-        - containerPort: 80
+          - /bin/sh
+          - "-c"
+          - "while true; do echo ping | nc -l -p 8000; done"
+          # - "while true; do sleep 10; done"
        volumeMounts:
        - mountPath: /data
          name: nfs-vol
+        ports:
+        - containerPort: 8000
      volumes:
      - name: nfs-vol
        nfs:
--- a/yml/hello.yml
+++ b/yml/hello.yml
@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello
+  labels:
+    run: hello
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8000
+    targetPort: 8000
+    protocol: TCP
+    name: http
+  selector:
+    app: hello
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+   name: hello
+   namespace: default
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: hello
+    spec:
+      containers:
+      - image: crccheck/hello-world
+        name: hello
+        # command:
+        #   - sleep
+        #   - "3600"
+        ports:
+        - containerPort: 8000
+
+
--- a/yml/traefik-conf.yml
+++ b/yml/traefik-conf.yml
@@ -0,0 +1,22 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: traefik-conf
+  namespace: kube-system
+data:
+  traefik.toml: |-
+    logLevel = "INFO"
+    defaultEntryPoints = ["http"]
+
+    [kubernetes]
+
+    [entryPoints]
+    [entryPoints.http]
+    address = ":80"
+    # [entryPoints.https]
+    # address = ":443"
+    # [entryPoints.https.tls]
+
+    [web]
+    address = ":8091"
--- a/yml/traefik-ui.yml
+++ b/yml/traefik-ui.yml
--- a/yml/traefik.yml
+++ b/yml/traefik.yml
@@ -0,0 +1,85 @@
+# ---
+# apiVersion: v1
+# kind: ServiceAccount
+# metadata:
+#   name: traefik-ingress-controller
+#   namespace: kube-system
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: traefik-ingress-controller
+  namespace: kube-system
+  labels:
+    k8s-app: traefik-ingress-lb
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: traefik-ingress-lb
+        name: traefik-ingress-lb
+    spec:
+      # serviceAccountName: traefik-ingress-controller
+      serviceAccountName: default
+      terminationGracePeriodSeconds: 60
+      hostNetwork: true
+      volumes:
+      - name: traefik-config
+        configMap:
+          name: traefik-conf
+      containers:
+      - image: traefik
+        name: traefik-ingress-lb
+        resources:
+          limits:
+            cpu: 200m
+            memory: 30Mi
+          requests:
+            cpu: 100m
+            memory: 20Mi
+        volumeMounts:
+          - mountPath: /etc/traefik
+            name: traefik-config
+        ports:
+        - name: http
+          containerPort: 80
+          hostPort: 80
+        - name: admin
+          containerPort: 8091
+        securityContext:
+          privileged: true
+        args:
+        - --web
+        - --web.address=:8091
+        - --kubernetes
+        - --configfile=/etc/traefik/traefik.toml
+        - --insecureSkipVerify=true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-web-ui
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector:
+    k8s-app: traefik-ingress-lb
+  ports:
+  - name: web
+    port: 8091
+    targetPort: 8091
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: traefik-web-ui
+  namespace: kube-system
+spec:
+  rules:
+  - host: traefik-ui.cluster.local
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: traefik-web-ui
+          servicePort: web