Works!

base/configuration.nix

@@ -16,7 +16,7 @@
   # Set your time zone.
   time.timeZone = "Europe/Oslo";
 
-  networking.search = [ "itpartner.intern" "itpartner.no" ];
+  networking.search = [ "itpartner.no" "itpartner.intern" ];
 
   services.openssh.enable = true;
   services.nfs.server.enable = true;

base/pki.nix

@@ -11,26 +11,17 @@ let
                   "expiry": "43800h",
                   "usages": [
                       "signing",
-                  "key encipherment",
+                      "key encipherment",
-                  "server auth"
+                      "server auth"
-                      ]
+                  ]
               },
               "client": {
                   "expiry": "43800h",
                   "usages": [
                       "signing",
-                  "key encipherment",
+                      "key encipherment",
-                  "client auth"
+                      "client auth"
-                      ]
+                  ]
-              },
-              "peer": {
-                  "expiry": "43800h",
-                  "usages": [
-                      "signing",
-                  "key encipherment",
-                  "server auth",
-                  "client auth"
-                      ]
               }
           }
       }
@@ -82,4 +73,52 @@ in
       buildInputs = [ pkgs.cfssl ];
     } (cfssl cert.csr cert.profile);
 
+  # server-cert = mkCert {
+  #     name = "kubernetes";
+  #     csr = csr {
+  #       cn = "kubernetes";
+  #       hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
+  #     };
+  #     profile = "server";
+  # };
+
+  # etcd0-cert = mkCert {
+  #   name = "etcd0";
+  #   csr = csr {
+  #     cn = "etcd0";
+  #     hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
+  #   };
+  #   profile = "peer";
+  # };
+
+  # etcd1-cert = mkCert {
+  #   name = "etcd1";
+  #   csr = csr {
+  #     cn = "etcd1";
+  #     hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
+  #   };
+  #   profile = "peer";
+  # };
+
+  # client-cert = mkCert {
+  #   name = "client";
+  #   csr = csr {
+  #     cn = "client";
+  #     hosts = '''';
+  #   };
+  #   profile = "client";
+  # };
+
+  # server_key = "${server-cert}/cert-key.pem";
+  # server_cert = "${server-cert}/cert.pem";
+
+  # etcd0_key = "${etcd0-cert}/cert-key.pem";
+  # etcd0_cert = "${etcd0-cert}/cert.pem";
+
+  # etcd1_key = "${etcd1-cert}/cert-key.pem";
+  # etcd1_cert = "${etcd1-cert}/cert.pem";
+
+  # client_key = "${client-cert}/cert-key.pem";
+  # client_cert = "${client-cert}/cert.pem";
+
   }

bin/teardown.sh

@@ -8,8 +8,9 @@ fi
 d=$1
 f=.$d.$$
 
-nixops ssh -d $d ${d}0-0 kubectl delete --all pods
+# nixops ssh -d $d ${d}0-0 kubectl delete --all pods
-nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
+# nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
+# sleep 60
 
 sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f
 nixops modify -d $d $f
@@ -17,4 +18,5 @@ nixops deploy -d $d
 rm $f
 
 nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"
+nixops ssh-for-each -d $d reboot
 

certs.nix

@@ -0,0 +1,149 @@
+let
+  pkgs = import <nixpkgs> {};
+
+  runWithOpenSSL = file: cmd: pkgs.runCommand file {
+    buildInputs = [ pkgs.openssl ];
+  } cmd;
+
+  etcd_cnf = pkgs.writeText "etcd-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = digitalSignature, keyEncipherment
+    extendedKeyUsage = serverAuth
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = etcd0
+    DNS.2 = etcd1
+    DNS.3 = etcd2
+    DNS.4 = k8s0-0
+    DNS.5 = k8s0-1
+    DNS.6 = k8s0-2
+    IP.1 = 127.0.0.1
+  '';
+
+  etcd_client_cnf = pkgs.writeText "etcd-client-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = digitalSignature, keyEncipherment
+    extendedKeyUsage = clientAuth
+  '';
+
+  apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = kubernetes
+    DNS.2 = kubernetes.default
+    DNS.3 = kubernetes.default.svc
+    DNS.4 = kubernetes.default.svc.cluster.local
+    IP.1 = 10.10.10.1
+    IP.2 = 10.253.18.100
+  '';
+
+  worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = k8s0-0
+    DNS.2 = k8s0-1
+    DNS.3 = k8s0-2
+  '';
+
+  ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
+  ca_pem = runWithOpenSSL "ca.pem" ''
+    openssl req \
+      -x509 -new -nodes -key ${ca_key} \
+      -days 10000 -out $out -subj "/CN=etcd-ca"
+  '';
+
+  etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
+  etcd_csr = runWithOpenSSL "etcd.csr" ''
+    openssl req \
+      -new -key ${etcd_key} \
+      -out $out -subj "/CN=etcd" \
+      -config ${etcd_cnf}
+  '';
+  etcd_cert = runWithOpenSSL "etcd.pem" ''
+    openssl x509 \
+      -req -in ${etcd_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} \
+      -CAcreateserial -out $out \
+      -days 365 -extensions v3_req \
+      -extfile ${etcd_cnf}
+  '';
+
+  etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
+    "openssl genrsa -out $out 2048";
+  etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''
+    openssl req \
+      -new -key ${etcd_client_key} \
+      -out $out -subj "/CN=etcd-client" \
+      -config ${etcd_client_cnf}
+  '';
+  etcd_client_cert = runWithOpenSSL "etcd-client.pem" ''
+    openssl x509 \
+      -req -in ${etcd_client_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${etcd_client_cnf}
+  '';
+
+  apiserver_key = runWithOpenSSL "apiserver-key.pem"
+    "openssl genrsa -out $out 2048";
+  apiserver_csr = runWithOpenSSL "apiserver.csr" ''
+    openssl req \
+      -new -key ${apiserver_key} \
+      -out $out -subj "/CN=kube-apiserver" \
+      -config ${apiserver_cnf}
+  '';
+  apiserver_cert = runWithOpenSSL "apiserver.pem" ''
+    openssl x509 \
+      -req -in ${apiserver_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${apiserver_cnf}
+  '';
+
+  worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
+  worker_csr = runWithOpenSSL "worker.csr" ''
+    openssl req \
+      -new -key ${worker_key} \
+      -out $out -subj "/CN=kube-worker" \
+      -config ${worker_cnf}
+  '';
+  worker_cert = runWithOpenSSL "worker.pem" ''
+    openssl x509 \
+      -req -in ${worker_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${worker_cnf}
+  '';
+
+in
+{
+  inherit ca_key ca_pem;
+  inherit etcd_key etcd_cert;
+  inherit etcd_client_key etcd_client_cert;
+  inherit apiserver_key apiserver_cert;
+  inherit worker_key worker_cert;
+}

git.nix

@@ -1,19 +0,0 @@
-{
-  git01 = { config, lib, pkgs, ... }:
-  {
-    deployment.targetHost = "10.253.18.103";
-    networking.hostName = "git01"; # Define your hostname
-    imports = [ ./hw/git01.nix ./git01/configuration.nix ];
-    services.nfs.server = {
-      enable=true;
-      exports= ''
-        /data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-        /data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-        /data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-        /data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-      '';
-    };
-    networking.firewall.allowedTCPPorts = [2049 111 20048];
-    networking.firewall.allowedUDPPorts = [2049 111 20048];
-  };
-}

k8s.nix

@@ -1,107 +1,57 @@
-with import ./base/pki.nix;
+with import ./certs.nix;
 let
-  server-cert = mkCert {
+  etcdServers = [ "etcd0" "etcd1" "etcd2" ];
-      name = "kubernetes";
+  # etcdServers = [ "k8s0-0" "k8s0-1" "k8s0-2" ];
-      csr = csr {
+  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
-        cn = "kubernetes";
+  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdServers;
-        hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
-      };
-      profile = "server";
-  };
 
-  etcd0-cert = mkCert {
+  etcdConfig = name: {
-    name = "etcd0";
-    csr = csr {
-      cn = "etcd0";
-      hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
-    };
-    profile = "peer";
-  };
-
-  etcd1-cert = mkCert {
-    name = "etcd1";
-    csr = csr {
-      cn = "etcd1";
-      hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
-    };
-    profile = "peer";
-  };
-
-  client-cert = mkCert {
-    name = "client";
-    csr = csr {
-      cn = "client";
-      hosts = '''';
-    };
-    profile = "client";
-  };
-
-  server_key = "${server-cert}/cert-key.pem";
-  server_cert = "${server-cert}/cert.pem";
-
-  etcd0_key = "${etcd0-cert}/cert-key.pem";
-  etcd0_cert = "${etcd0-cert}/cert.pem";
-
-  etcd1_key = "${etcd1-cert}/cert-key.pem";
-  etcd1_cert = "${etcd1-cert}/cert.pem";
-
-  client_key = "${client-cert}/cert-key.pem";
-  client_cert = "${client-cert}/cert.pem";
-
-  etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ];
-  etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ];
-
-  etcdConfig = etcd: {
     services.etcd = {
-      name = etcd.name;
+      inherit name;
       enable = true;
       listenClientUrls = ["https://0.0.0.0:2379"];
       listenPeerUrls = ["https://0.0.0.0:2380"];
       peerClientCertAuth = true;
-      keyFile  = "${etcd.key}";
+      keyFile  = etcd_key;
-      certFile = "${etcd.cert}";
+      certFile = etcd_cert;
-      trustedCaFile = "${ca_cert}";
+      trustedCaFile = ca_pem;
-      advertiseClientUrls = [ "https://${etcd.name}:2379" ];
+      advertiseClientUrls = [ "https://${name}:2379" ];
-      initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ];
+      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
       initialCluster = etcdCluster;
     };
     environment.variables = {
-      ETCDCTL_KEY_FILE  = "${etcd.key}";
+      ETCDCTL_KEY_FILE  = "${etcd_client_key}";
-      ETCDCTL_CERT_FILE = "${etcd.cert}";
+      ETCDCTL_CERT_FILE = "${etcd_client_cert}";
-      ETCDCTL_CA_FILE   = "${ca_cert}";
+      ETCDCTL_CA_FILE   = "${ca_pem}";
-      ETCDCTL_PEERS     = "https://localhost:2379";
+      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
     };
-    # networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
     systemd.services.flannel.after = [ "etcd.service" ];
   };
 
-  flannelConfig = {
+  kubeConfig = {
     services.flannel = {
       enable = true;
       network = "10.10.0.0/16";
-      iface = "enp2s0";
+      iface = "ens32";
       etcd = {
-        endpoints = etcdServers;
+        endpoints = etcdEndpoints;
-        caFile   = "${ca_cert}";
+        keyFile  = etcd_client_key;
-        keyFile  = "${client_key}";
+        certFile = etcd_client_cert;
-        certFile = "${client_cert}";
+        caFile   = ca_pem;
       };
     };
-    # networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
+    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
-  };
-
-  kubeConfig = {
     systemd.services.docker = {
       after = [ "flannel.service" ];
       serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
     };
-    virtualisation.docker.extraOptions =
+    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU";
-      "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
     services.kubernetes.etcd = {
-      servers = etcdServers;
+      servers = etcdEndpoints;
-      caFile   = "${ca_cert}";
+      keyFile  = etcd_client_key;
-      keyFile  = "${client_key}";
+      certFile = etcd_client_cert;
-      certFile = "${client_cert}";
+      caFile   = ca_pem;
     };
     # services.kubernetes.verbose = true;
   };
@@ -110,26 +60,26 @@ let
     services.kubernetes = {
       roles = [ "node" ];
       kubeconfig = {
-        server = "https://10.253.18.100:443";
+        server = "https://10.253.18.100:4443";
-        caFile   = "${ca_cert}";
+        keyFile  = worker_key;
-        keyFile  = "${client_key}";
+        certFile = worker_cert;
-        certFile = "${client_cert}";
+        caFile   = ca_pem;
       };
       kubelet = {
-        tlsKeyFile  = "${client_key}";
+        tlsKeyFile  = worker_key;
-        tlsCertFile = "${client_cert}";
+        tlsCertFile = worker_cert;
         networkPlugin = null;
-        clusterDns = "10.10.21.0";
+        clusterDns = "10.253.18.100";
       };
     };
-    networking.firewall = {
+    networking = {
-      enable = false;
+      firewall = {
-      # allowedTCPPorts = [ 53 10250 8000 8080 ];
+        enable = true;
-      # allowedUDPPorts = [ 53 ];
+        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
-      # trustedInterfaces = [ "flannel.1" "docker0" ];
+        allowedTCPPorts = [ 53 10250 ];
-      # extraCommands = ''
+        allowedUDPPorts = [ 53 ];
-      #    iptables -P FORWARD ACCEPT
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
-      # '';
+      };
     };
   };
 
@@ -137,44 +87,46 @@ let
     services.kubernetes = {
       roles = [ "master" ];
       apiserver = {
-        publicAddress = "0.0.0.0";
         address = "0.0.0.0";
-        clientCaFile = "${ca_cert}";
+        publicAddress = "0.0.0.0";
-        tlsKeyFile   = "${server_key}";
+        advertiseAddress = "10.253.18.100";
-        tlsCertFile  = "${server_cert}";
+        securePort = 4443;
-        # serviceAccountKeyFile = "${server_key}";
+        tlsKeyFile   = apiserver_key;
-        # kubeletClientCaFile   = "${ca_cert}";
+        tlsCertFile  = apiserver_cert;
-        # kubeletClientKeyFile  = "${client_key}";
+        clientCaFile = ca_pem;
-        # kubeletClientCertFile = "${client_cert}";
+        kubeletClientCaFile   = ca_pem;
+        kubeletClientKeyFile  = worker_key;
+        kubeletClientCertFile = worker_cert;
+        # serviceAccountKeyFile = apiserver_key;
       };
       scheduler.leaderElect = true;
       controllerManager.leaderElect = true;
-      controllerManager.serviceAccountKeyFile = "${server_key}";
+      controllerManager.serviceAccountKeyFile = apiserver_key;
+      dns.enable = true;
+      dns.port = 4053;
+    };
+    networking.firewall = {
+      allowedTCPPorts = [ 5000 8080 4443 4053 ];
+      allowedUDPPorts = [ 4053 ];
     };
-    # networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
-    # networking.firewall.allowedUDPPorts = [ 53 ];
   };
 
   baseConfig = node: {
     imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
-    networking.hostName = node;
+    networking = {
-    networking.extraHosts = ''
+      hostName = node;
-      10.253.18.100 etcd0 k8s0-0 kubernetes
+      extraHosts = ''
-      10.253.18.101 etcd1
+        10.253.18.100 etcd0 kubernetes
-    '';
+        10.253.18.101 etcd1
-    virtualisation.docker.enable = true;
+        10.253.18.102 etcd2
-  };
+      '';
-
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
-  etcdConf0 = etcdConfig {
+      firewall.allowedTCPPorts = [ 80 443 ];
-    name = "etcd0";
+    };
-    key = etcd0_key;
+    services.dnsmasq.enable = true;
-    cert = etcd0_cert;
+    services.dnsmasq.servers = [
-  };
+      "/cluster.local/10.253.18.100#4053"
-
+    ];
-  etcdConf1 = etcdConfig {
-    name = "etcd1";
-    key = etcd1_key;
-    cert = etcd1_cert;
   };
 
   minion = host: ip: { config, lib, pkgs, ... }:
@@ -184,18 +136,19 @@ let
     in
     {
       deployment.targetHost = ip;
-      require = [ base flannelConfig kubeConfig kubeNode ];
+      require = [ base kubeConfig kubeNode ];
-  };
+      services.kubernetes.dns.enable = false;
+    };
 in
 {
   k8s0-0 = { config, lib, pkgs, ... }:
     let
       base = baseConfig "k8s0-0";
-      etcd = etcdConf0;
+      etcd = etcdConfig "etcd0";
     in
     {
       deployment.targetHost = "10.253.18.100";
-      require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ];
+      require = [ base etcd kubeConfig kubeMaster kubeNode ];
       services.dockerRegistry = {
         enable = true;
         listenAddress = "0.0.0.0";
@@ -205,12 +158,22 @@ in
   k8s0-1 = { config, lib, pkgs, ... }:
     let
       base = baseConfig "k8s0-1";
-      etcd = etcdConf1;
+      etcd = etcdConfig "etcd1";
     in
     {
       deployment.targetHost = "10.253.18.101";
-      require = [ base etcd flannelConfig kubeConfig kubeNode ];
+      require = [ base etcd kubeConfig kubeNode ];
-  };
+      services.kubernetes.dns.enable = false;
+    };
 
-  k8s0-2 = minion "k8s0-2" "10.253.18.102";
+  k8s0-2 = { config, lib, pkgs, ... }:
+    let
+      base = baseConfig "k8s0-2";
+      etcd = etcdConfig "etcd2";
+    in
+    {
+      deployment.targetHost = "10.253.18.102";
+      require = [ base etcd kubeConfig kubeNode ];
+      services.kubernetes.dns.enable = false;
+    };
 }

test/gitlab.yaml

@@ -1,56 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gitlab
-  labels:
-    run: gitlab
-spec:
-  type: NodePort
-  ports:
-  - port: 80
-    targetPort: 80
-    protocol: TCP
-    name: http
-  - port: 443
-    protocol: TCP
-    name: https
-  - port: 22
-    protocol: TCP
-    name: ssh
-  selector:
-   run: gitlab
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: gitlab
-  namespace: default
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: busybox
-      containers:
-      - image: gitlab/gitlab-ce:latest
-        name: gitlab-container
-        ports:
-        - containerPort: 80
-        - containerPort: 22
-        - containerPort: 443
-      volumeMounts:
-      - mountPath: /etc/gitlab
-        name: gitlab-data
-        subPath: config
-      - mountPath: /var/log/gitlab
-        name: gitlab-data
-        subPath: logs
-      - mountPath: /var/opt/gitlab
-        name: gitlab-data
-        subPath: data
-      volumes:
-      - name: gitlab-data
-        nfs:
-          server: 10.253.10.103
-          path: /data/gitlab
-          ReadOnly: false

yml/busybox.yml

@@ -5,17 +5,14 @@ metadata:
   labels:
     run: bbox
 spec:
-  type: NodePort
+  type: ClusterIP
+  selector:
+   app: busybox
   ports:
-  - port: 8080
+  - port: 8000
-    targetPort: 80
+    targetPort: 8000
     protocol: TCP
     name: http
-  - port: 443
-    protocol: TCP
-    name: https
-  selector:
-   run: bbox
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -33,13 +30,15 @@ spec:
       - image: busybox
         name: busybox
         command:
-          - sleep
+          - /bin/sh
-          - "3600"
+          - "-c"
-        ports:
+          - "while true; do echo ping | nc -l -p 8000; done"
-        - containerPort: 80
+          # - "while true; do sleep 10; done"
         volumeMounts:
         - mountPath: /data
           name: nfs-vol
+        ports:
+        - containerPort: 8000
       volumes:
       - name: nfs-vol
         nfs:

yml/hello.yml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello
+  labels:
+    run: hello
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8000
+    targetPort: 8000
+    protocol: TCP
+    name: http
+  selector:
+    app: hello
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+   name: hello
+   namespace: default
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: hello
+    spec:
+      containers:
+      - image: crccheck/hello-world
+        name: hello
+        # command:
+        #   - sleep
+        #   - "3600"
+        ports:
+        - containerPort: 8000
+
+

yml/traefik-conf.yml

@@ -0,0 +1,22 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: traefik-conf
+  namespace: kube-system
+data:
+  traefik.toml: |-
+    logLevel = "INFO"
+    defaultEntryPoints = ["http"]
+
+    [kubernetes]
+
+    [entryPoints]
+    [entryPoints.http]
+    address = ":80"
+    # [entryPoints.https]
+    # address = ":443"
+    # [entryPoints.https.tls]
+
+    [web]
+    address = ":8091"

yml/traefik.yml

@@ -0,0 +1,85 @@
+# ---
+# apiVersion: v1
+# kind: ServiceAccount
+# metadata:
+#   name: traefik-ingress-controller
+#   namespace: kube-system
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: traefik-ingress-controller
+  namespace: kube-system
+  labels:
+    k8s-app: traefik-ingress-lb
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: traefik-ingress-lb
+        name: traefik-ingress-lb
+    spec:
+      # serviceAccountName: traefik-ingress-controller
+      serviceAccountName: default
+      terminationGracePeriodSeconds: 60
+      hostNetwork: true
+      volumes:
+      - name: traefik-config
+        configMap:
+          name: traefik-conf
+      containers:
+      - image: traefik
+        name: traefik-ingress-lb
+        resources:
+          limits:
+            cpu: 200m
+            memory: 30Mi
+          requests:
+            cpu: 100m
+            memory: 20Mi
+        volumeMounts:
+          - mountPath: /etc/traefik
+            name: traefik-config
+        ports:
+        - name: http
+          containerPort: 80
+          hostPort: 80
+        - name: admin
+          containerPort: 8091
+        securityContext:
+          privileged: true
+        args:
+        - --web
+        - --web.address=:8091
+        - --kubernetes
+        - --configfile=/etc/traefik/traefik.toml
+        - --insecureSkipVerify=true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-web-ui
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector:
+    k8s-app: traefik-ingress-lb
+  ports:
+  - name: web
+    port: 8091
+    targetPort: 8091
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: traefik-web-ui
+  namespace: kube-system
+spec:
+  rules:
+  - host: traefik-ui.cluster.local
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: traefik-web-ui
+          servicePort: web