Major revamp.
This commit is contained in:
Jonas Juselius
124
base/pki.nix
124
base/pki.nix
@@ -1,124 +0,0 @@
|
|||||||
with import <nixpkgs> {};
|
|
||||||
let
|
|
||||||
ca-config = pkgs.writeText "ca-config.json" ''
|
|
||||||
{
|
|
||||||
"signing": {
|
|
||||||
"default": {
|
|
||||||
"expiry": "43800h"
|
|
||||||
},
|
|
||||||
"profiles": {
|
|
||||||
"server": {
|
|
||||||
"expiry": "43800h",
|
|
||||||
"usages": [
|
|
||||||
"signing",
|
|
||||||
"key encipherment",
|
|
||||||
"server auth"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"client": {
|
|
||||||
"expiry": "43800h",
|
|
||||||
"usages": [
|
|
||||||
"signing",
|
|
||||||
"key encipherment",
|
|
||||||
"client auth"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
'';
|
|
||||||
|
|
||||||
csr = args: pkgs.writeText "${args.cn}-cert.json" ''
|
|
||||||
{
|
|
||||||
"CN": "${args.cn}",
|
|
||||||
"hosts": [ ${args.hosts} ],
|
|
||||||
"key": {
|
|
||||||
"algo": "rsa",
|
|
||||||
"size": 2048
|
|
||||||
},
|
|
||||||
"names": [
|
|
||||||
{
|
|
||||||
"C": "NO",
|
|
||||||
"L": "Tromsø",
|
|
||||||
"O": "Serit IT Partner Tromsø AS",
|
|
||||||
"OU": "",
|
|
||||||
"ST": ""
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
'';
|
|
||||||
|
|
||||||
ca-csr = csr { cn = "kubernetes"; hosts = ""; };
|
|
||||||
ca = pkgs.runCommand "ca-cert" {
|
|
||||||
buildInputs = [ pkgs.cfssl ];
|
|
||||||
} '' cfssl genkey -initca ${ca-csr} | cfssljson -bare ca; \
|
|
||||||
mkdir -p $out; cp *.pem $out'';
|
|
||||||
|
|
||||||
ca_cert = "${ca}/ca.pem";
|
|
||||||
ca_key = "${ca}/ca-key.pem";
|
|
||||||
|
|
||||||
cfssl = name: profile: ''
|
|
||||||
cfssl gencert -ca ${ca_cert} -ca-key ${ca_key} \
|
|
||||||
-config=${ca-config} -profile=${profile} ${name} | cfssljson -bare cert; \
|
|
||||||
mkdir -p $out; cp *.pem $out
|
|
||||||
'';
|
|
||||||
in
|
|
||||||
rec {
|
|
||||||
inherit ca_cert;
|
|
||||||
inherit ca_key;
|
|
||||||
inherit csr;
|
|
||||||
|
|
||||||
mkCert = cert:
|
|
||||||
pkgs.runCommand "${cert.name}-cert" {
|
|
||||||
buildInputs = [ pkgs.cfssl ];
|
|
||||||
} (cfssl cert.csr cert.profile);
|
|
||||||
|
|
||||||
# server-cert = mkCert {
|
|
||||||
# name = "kubernetes";
|
|
||||||
# csr = csr {
|
|
||||||
# cn = "kubernetes";
|
|
||||||
# hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
|
|
||||||
# };
|
|
||||||
# profile = "server";
|
|
||||||
# };
|
|
||||||
|
|
||||||
# etcd0-cert = mkCert {
|
|
||||||
# name = "etcd0";
|
|
||||||
# csr = csr {
|
|
||||||
# cn = "etcd0";
|
|
||||||
# hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
|
|
||||||
# };
|
|
||||||
# profile = "peer";
|
|
||||||
# };
|
|
||||||
|
|
||||||
# etcd1-cert = mkCert {
|
|
||||||
# name = "etcd1";
|
|
||||||
# csr = csr {
|
|
||||||
# cn = "etcd1";
|
|
||||||
# hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
|
|
||||||
# };
|
|
||||||
# profile = "peer";
|
|
||||||
# };
|
|
||||||
|
|
||||||
# client-cert = mkCert {
|
|
||||||
# name = "client";
|
|
||||||
# csr = csr {
|
|
||||||
# cn = "client";
|
|
||||||
# hosts = '''';
|
|
||||||
# };
|
|
||||||
# profile = "client";
|
|
||||||
# };
|
|
||||||
|
|
||||||
# server_key = "${server-cert}/cert-key.pem";
|
|
||||||
# server_cert = "${server-cert}/cert.pem";
|
|
||||||
|
|
||||||
# etcd0_key = "${etcd0-cert}/cert-key.pem";
|
|
||||||
# etcd0_cert = "${etcd0-cert}/cert.pem";
|
|
||||||
|
|
||||||
# etcd1_key = "${etcd1-cert}/cert-key.pem";
|
|
||||||
# etcd1_cert = "${etcd1-cert}/cert.pem";
|
|
||||||
|
|
||||||
# client_key = "${client-cert}/cert-key.pem";
|
|
||||||
# client_cert = "${client-cert}/cert.pem";
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -17,6 +17,6 @@ nixops modify -d $d $f
|
|||||||
nixops deploy -d $d
|
nixops deploy -d $d
|
||||||
rm $f
|
rm $f
|
||||||
|
|
||||||
|
nixops reboot -d $d
|
||||||
nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"
|
nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"
|
||||||
nixops ssh-for-each -d $d reboot
|
|
||||||
|
|
||||||
|
|||||||
63
fs0.nix
Normal file
63
fs0.nix
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
with import <nixpkgs> {};
|
||||||
|
let
|
||||||
|
pki = pkgs.callPackage ./lib/pki.nix {};
|
||||||
|
certs = {
|
||||||
|
ca = pki.ca;
|
||||||
|
fs = pki.etcd ''
|
||||||
|
"fs0-0",
|
||||||
|
"fs0-1",
|
||||||
|
"10.253.18.106",
|
||||||
|
"10.1.2.164",
|
||||||
|
"127.0.0.1"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
clusterHosts = ''
|
||||||
|
10.253.18.106 fs0-0
|
||||||
|
10.1.2.164 fs0-1
|
||||||
|
'';
|
||||||
|
|
||||||
|
nixosConfig = node: {
|
||||||
|
imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ];
|
||||||
|
networking = {
|
||||||
|
hostName = node;
|
||||||
|
extraHosts = clusterHosts;
|
||||||
|
# firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
||||||
|
# firewall.allowedTCPPorts = [ 80 443 ];
|
||||||
|
};
|
||||||
|
environment.systemPackages = [ pkgs.tshark ];
|
||||||
|
services.glusterfs = {
|
||||||
|
enable = true;
|
||||||
|
tlsSettings = {
|
||||||
|
caCert = certs.ca.cert;
|
||||||
|
tlsKeyPath = certs.fs.key;
|
||||||
|
tlsPem = certs.fs.cert;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
fs0-0 = { ... }:
|
||||||
|
let
|
||||||
|
base = nixosConfig "fs0-0";
|
||||||
|
in
|
||||||
|
{
|
||||||
|
deployment.targetHost = "10.253.18.106";
|
||||||
|
require = [ base ];
|
||||||
|
services.nfs.server = {
|
||||||
|
enable=true;
|
||||||
|
exports= ''
|
||||||
|
/data/vol1 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networking.firewall.allowedTCPPorts = [ 111 2049 ];
|
||||||
|
networking.firewall.allowedUDPPorts = [ 111 2049 ];
|
||||||
|
};
|
||||||
|
fs0-1 = { ... }:
|
||||||
|
let
|
||||||
|
base = nixosConfig "fs0-1";
|
||||||
|
in
|
||||||
|
{
|
||||||
|
deployment.targetHost = "10.1.2.164";
|
||||||
|
require = [ base ];
|
||||||
|
};
|
||||||
|
}
|
||||||
17
git.nix
17
git.nix
@@ -1,17 +0,0 @@
|
|||||||
{
|
|
||||||
git01 = { config, lib, pkgs, ... }:
|
|
||||||
{
|
|
||||||
deployment.targetHost = "10.253.18.103";
|
|
||||||
networking.hostName = "git01"; # Define your hostname
|
|
||||||
imports = [ ./hw/git01.nix ./base/configuration.nix ];
|
|
||||||
services.nfs.server = {
|
|
||||||
enable=true;
|
|
||||||
exports= ''
|
|
||||||
/data 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
|
||||||
/vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
networking.firewall.allowedTCPPorts = [2049 111 20048];
|
|
||||||
networking.firewall.allowedUDPPorts = [2049 111 20048];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
29
hardware-configuration/fs0-0.nix
Normal file
29
hardware-configuration/fs0-0.nix
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||||
|
# and may be overwritten by future invocations. Please make changes
|
||||||
|
# to /etc/nixos/configuration.nix instead.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [ ];
|
||||||
|
|
||||||
|
boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
|
||||||
|
boot.kernelModules = [ ];
|
||||||
|
boot.extraModulePackages = [ ];
|
||||||
|
|
||||||
|
fileSystems."/" =
|
||||||
|
{ device = "/dev/disk/by-uuid/6b3d4c49-9719-49b3-8210-d53374cd0eff";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
fileSystems."/var/log" =
|
||||||
|
{ device = "/dev/disk/by-uuid/c1e78683-4fde-4029-a9f3-7631df649b2f";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
fileSystems."/data" =
|
||||||
|
{ device = "/dev/sdb1";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
|
swapDevices = [ ];
|
||||||
|
|
||||||
|
nix.maxJobs = lib.mkDefault 1;
|
||||||
|
}
|
||||||
26
hardware-configuration/fs0-1.nix
Normal file
26
hardware-configuration/fs0-1.nix
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||||
|
# and may be overwritten by future invocations. Please make changes
|
||||||
|
# to /etc/nixos/configuration.nix instead.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [ ];
|
||||||
|
|
||||||
|
boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
|
||||||
|
boot.kernelModules = [ ];
|
||||||
|
boot.extraModulePackages = [ ];
|
||||||
|
|
||||||
|
fileSystems."/" =
|
||||||
|
{ device = "/dev/disk/by-uuid/e8820516-9c21-46f4-9dde-a7a77bf67bbd";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
fileSystems."/var/log" =
|
||||||
|
{ device = "/dev/disk/by-uuid/c130b88c-0699-4836-b967-47bdee0a5453";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
|
swapDevices = [ ];
|
||||||
|
|
||||||
|
nix.maxJobs = lib.mkDefault 1;
|
||||||
|
}
|
||||||
|
|
||||||
21
hardware-configuration/k0-4.nix
Normal file
21
hardware-configuration/k0-4.nix
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||||
|
# and may be overwritten by future invocations. Please make changes
|
||||||
|
# to /etc/nixos/configuration.nix instead.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [ ];
|
||||||
|
|
||||||
|
boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
|
||||||
|
boot.kernelModules = [ ];
|
||||||
|
boot.extraModulePackages = [ ];
|
||||||
|
|
||||||
|
fileSystems."/" =
|
||||||
|
{ device = "/dev/disk/by-uuid/241cdf57-a2b5-482d-a522-01725badc859";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
|
swapDevices = [ ];
|
||||||
|
|
||||||
|
nix.maxJobs = lib.mkDefault 1;
|
||||||
|
}
|
||||||
21
hardware-configuration/k0-5.nix
Normal file
21
hardware-configuration/k0-5.nix
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||||
|
# and may be overwritten by future invocations. Please make changes
|
||||||
|
# to /etc/nixos/configuration.nix instead.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [ ];
|
||||||
|
|
||||||
|
boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
|
||||||
|
boot.kernelModules = [ ];
|
||||||
|
boot.extraModulePackages = [ ];
|
||||||
|
|
||||||
|
fileSystems."/" =
|
||||||
|
{ device = "/dev/disk/by-uuid/67441b95-19f2-484d-b57b-3f4b2a55f3cc";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
|
swapDevices = [ ];
|
||||||
|
|
||||||
|
nix.maxJobs = lib.mkDefault 1;
|
||||||
|
}
|
||||||
21
hardware-configuration/k1-0.nix
Normal file
21
hardware-configuration/k1-0.nix
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||||
|
# and may be overwritten by future invocations. Please make changes
|
||||||
|
# to /etc/nixos/configuration.nix instead.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [ ];
|
||||||
|
|
||||||
|
boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
|
||||||
|
boot.kernelModules = [ ];
|
||||||
|
boot.extraModulePackages = [ ];
|
||||||
|
|
||||||
|
fileSystems."/" =
|
||||||
|
{ device = "/dev/disk/by-uuid/2e7ba83d-014f-4ef5-a1ce-fc9e34ce7b83";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
|
swapDevices = [ ];
|
||||||
|
|
||||||
|
nix.maxJobs = lib.mkDefault 1;
|
||||||
|
}
|
||||||
21
hardware-configuration/k1-1.nix
Normal file
21
hardware-configuration/k1-1.nix
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||||
|
# and may be overwritten by future invocations. Please make changes
|
||||||
|
# to /etc/nixos/configuration.nix instead.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [ ];
|
||||||
|
|
||||||
|
boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
|
||||||
|
boot.kernelModules = [ ];
|
||||||
|
boot.extraModulePackages = [ ];
|
||||||
|
|
||||||
|
fileSystems."/" =
|
||||||
|
{ device = "/dev/disk/by-uuid/70b9d730-9cb6-48e2-8e00-fa78c8feefdf";
|
||||||
|
fsType = "ext4";
|
||||||
|
};
|
||||||
|
|
||||||
|
swapDevices = [ ];
|
||||||
|
|
||||||
|
nix.maxJobs = lib.mkDefault 1;
|
||||||
|
}
|
||||||
74
k0.nix
Normal file
74
k0.nix
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
with import <nixpkgs> {};
|
||||||
|
let
|
||||||
|
pki = pkgs.callPackage ./lib/pki.nix {};
|
||||||
|
certs = {
|
||||||
|
ca = pki.ca;
|
||||||
|
apiserver = pki.apiserver ''
|
||||||
|
"10.253.18.100",
|
||||||
|
"10.0.0.1",
|
||||||
|
"127.0.0.1",
|
||||||
|
"kubernetes",
|
||||||
|
"etcd0",
|
||||||
|
"k0-0"
|
||||||
|
'';
|
||||||
|
kube-proxy = pki.kube-proxy;
|
||||||
|
admin = pki.admin;
|
||||||
|
etcd = pki.etcd ''
|
||||||
|
"etcd0",
|
||||||
|
"etcd1",
|
||||||
|
"etcd2",
|
||||||
|
"10.253.18.100",
|
||||||
|
"10.253.18.101",
|
||||||
|
"10.253.18.102",
|
||||||
|
"127.0.0.1"
|
||||||
|
'';
|
||||||
|
k0-0 = pki.worker { name = "k0-0"; ip = "10.253.18.100"; };
|
||||||
|
k0-1 = pki.worker { name = "k0-1"; ip = "10.253.18.101"; };
|
||||||
|
k0-2 = pki.worker { name = "k0-2"; ip = "10.253.18.102"; };
|
||||||
|
k0-3 = pki.worker { name = "k0-3"; ip = "10.253.18.103"; };
|
||||||
|
k0-4 = pki.worker { name = "k0-4"; ip = "10.253.18.107"; };
|
||||||
|
k0-5 = pki.worker { name = "k0-5"; ip = "10.253.18.108"; };
|
||||||
|
};
|
||||||
|
cluster = callPackage ./lib/k8s.nix {
|
||||||
|
kubeMaster = "10.253.18.100";
|
||||||
|
etcdNodes = [ "etcd0" "etcd1" "etcd2" ];
|
||||||
|
clusterHosts = ''
|
||||||
|
10.253.18.100 k0-0 etcd0 kubernetes
|
||||||
|
10.253.18.101 k0-1 etcd1
|
||||||
|
10.253.18.102 k0-2 etcd2
|
||||||
|
10.253.18.103 k0-3
|
||||||
|
10.253.18.107 k0-4
|
||||||
|
10.253.18.108 k0-5
|
||||||
|
10.253.18.106 fs0-0
|
||||||
|
10.1.2.164 fs0-1
|
||||||
|
'';
|
||||||
|
inherit certs;
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
k0-0 = cluster.apiserver "k0-0" "10.253.18.100" "etcd0";
|
||||||
|
k0-1 = cluster.server "k0-1" "10.253.18.101" "etcd1";
|
||||||
|
k0-2 = cluster.server "k0-2" "10.253.18.102" "etcd2";
|
||||||
|
k0-3 = cluster.worker "k0-3" "10.253.18.103";
|
||||||
|
k0-4 = cluster.worker "k0-4" "10.253.18.107";
|
||||||
|
k0-5 = cluster.worker "k0-5" "10.253.18.108";
|
||||||
|
|
||||||
|
# k0-3 = { config, lib, pkgs, ... }:
|
||||||
|
# let
|
||||||
|
# instance = "k0-3";
|
||||||
|
# base = k8s.nixosConfig instance;
|
||||||
|
# in
|
||||||
|
# {
|
||||||
|
# deployment.targetHost = "10.253.18.103";
|
||||||
|
# require = [ k8s.base (k8s.kubeConfig instance) (k8s.kubeNode instance) ];
|
||||||
|
# services.kubernetes.addons.dns.enable = false;
|
||||||
|
# services.nfs.server = {
|
||||||
|
# enable=true;
|
||||||
|
# exports= ''
|
||||||
|
# /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||||
|
# '';
|
||||||
|
# };
|
||||||
|
# networking.firewall.allowedTCPPorts = [ 111 2049 ];
|
||||||
|
# networking.firewall.allowedUDPPorts = [ 111 2049 ];
|
||||||
|
# };
|
||||||
|
}
|
||||||
209
k8s.nix
209
k8s.nix
@@ -1,209 +0,0 @@
|
|||||||
with import ./certs.nix;
|
|
||||||
let
|
|
||||||
pkgs = import <nixpkgs> {};
|
|
||||||
|
|
||||||
kube_apiserver = "https://10.253.18.100:443";
|
|
||||||
etcdServers = [ "etcd0" "etcd1" "etcd2" ];
|
|
||||||
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
|
|
||||||
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers;
|
|
||||||
|
|
||||||
etcdConfig = name: {
|
|
||||||
services.etcd = {
|
|
||||||
inherit name;
|
|
||||||
enable = true;
|
|
||||||
listenClientUrls = ["https://0.0.0.0:2379"];
|
|
||||||
listenPeerUrls = ["https://0.0.0.0:2380"];
|
|
||||||
peerClientCertAuth = true;
|
|
||||||
keyFile = etcd_key;
|
|
||||||
certFile = etcd_cert;
|
|
||||||
trustedCaFile = ca_pem;
|
|
||||||
advertiseClientUrls = [ "https://${name}:2379" ];
|
|
||||||
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
|
|
||||||
initialCluster = etcdCluster;
|
|
||||||
};
|
|
||||||
environment.variables = {
|
|
||||||
ETCDCTL_KEY_FILE = "${etcd_client_key}";
|
|
||||||
ETCDCTL_CERT_FILE = "${etcd_client_cert}";
|
|
||||||
ETCDCTL_CA_FILE = "${ca_pem}";
|
|
||||||
ETCDCTL_PEERS = "https://127.0.0.1:2379";
|
|
||||||
};
|
|
||||||
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
|
|
||||||
systemd.services.flannel.after = [ "etcd.service" ];
|
|
||||||
};
|
|
||||||
|
|
||||||
kubeconfig = {
|
|
||||||
caFile = ca_pem;
|
|
||||||
keyFile = worker_key;
|
|
||||||
certFile = worker_cert;
|
|
||||||
server = kube_apiserver;
|
|
||||||
};
|
|
||||||
|
|
||||||
kubeNode = {
|
|
||||||
services.kubernetes = {
|
|
||||||
roles = [ "node" ];
|
|
||||||
kubeconfig = {
|
|
||||||
server = kube_apiserver;
|
|
||||||
keyFile = worker_key;
|
|
||||||
certFile = worker_cert;
|
|
||||||
caFile = ca_pem;
|
|
||||||
};
|
|
||||||
kubelet = {
|
|
||||||
enable = true;
|
|
||||||
clientCaFile = ca_pem;
|
|
||||||
tlsKeyFile = worker_key;
|
|
||||||
tlsCertFile = worker_cert;
|
|
||||||
networkPlugin = null;
|
|
||||||
# clusterDns = "10.253.18.100";
|
|
||||||
clusterDns = "10.0.0.254";
|
|
||||||
inherit kubeconfig;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
networking = {
|
|
||||||
firewall = {
|
|
||||||
enable = true;
|
|
||||||
# trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
|
|
||||||
allowedTCPPorts = [ 53 4194 10250 ];
|
|
||||||
allowedUDPPorts = [ 53 ];
|
|
||||||
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
|
||||||
};
|
|
||||||
|
|
||||||
kubeMaster = {
|
|
||||||
services.kubernetes = {
|
|
||||||
roles = [ "master" ];
|
|
||||||
kubelet.unschedulable = true;
|
|
||||||
apiserver = {
|
|
||||||
address = "0.0.0.0";
|
|
||||||
publicAddress = "0.0.0.0";
|
|
||||||
advertiseAddress = "10.253.18.100";
|
|
||||||
securePort = 443;
|
|
||||||
tlsKeyFile = apiserver_key;
|
|
||||||
tlsCertFile = apiserver_cert;
|
|
||||||
clientCaFile = ca_pem;
|
|
||||||
kubeletClientCaFile = ca_pem;
|
|
||||||
kubeletClientKeyFile = worker_key;
|
|
||||||
kubeletClientCertFile = worker_cert;
|
|
||||||
serviceAccountKeyFile = apiserver_key;
|
|
||||||
};
|
|
||||||
scheduler.leaderElect = true;
|
|
||||||
controllerManager = {
|
|
||||||
leaderElect = true;
|
|
||||||
serviceAccountKeyFile = apiserver_key;
|
|
||||||
rootCaFile = ca_pem;
|
|
||||||
inherit kubeconfig;
|
|
||||||
};
|
|
||||||
addons.dashboard.enable = true;
|
|
||||||
addons.dns.enable = true;
|
|
||||||
};
|
|
||||||
networking.firewall = {
|
|
||||||
allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
|
|
||||||
# allowedUDPPorts = [ 4053 ];
|
|
||||||
};
|
|
||||||
environment.systemPackages = [ pkgs.kubernetes-helm ];
|
|
||||||
};
|
|
||||||
|
|
||||||
kubeConfig = {
|
|
||||||
services.kubernetes = {
|
|
||||||
verbose = false;
|
|
||||||
caFile = ca_pem;
|
|
||||||
flannel.enable = true;
|
|
||||||
clusterCidr = "10.10.0.0/16";
|
|
||||||
etcd = {
|
|
||||||
servers = etcdEndpoints;
|
|
||||||
keyFile = etcd_client_key;
|
|
||||||
certFile = etcd_client_cert;
|
|
||||||
caFile = ca_pem;
|
|
||||||
};
|
|
||||||
proxy = {
|
|
||||||
inherit kubeconfig;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
baseConfig = node: {
|
|
||||||
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
|
|
||||||
networking = {
|
|
||||||
hostName = node;
|
|
||||||
extraHosts = ''
|
|
||||||
10.253.18.100 etcd0 kubernetes
|
|
||||||
10.253.18.101 etcd1
|
|
||||||
10.253.18.102 etcd2
|
|
||||||
'';
|
|
||||||
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
|
||||||
firewall.allowedTCPPorts = [ 80 443 ];
|
|
||||||
};
|
|
||||||
environment.systemPackages = [ pkgs.tshark ];
|
|
||||||
};
|
|
||||||
|
|
||||||
minion = host: ip: { config, lib, pkgs, ... }:
|
|
||||||
let
|
|
||||||
inherit host;
|
|
||||||
base = baseConfig host;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
deployment.targetHost = ip;
|
|
||||||
require = [ base kubeConfig kubeNode ];
|
|
||||||
services.kubernetes.addons.dns.enable = false;
|
|
||||||
};
|
|
||||||
in
|
|
||||||
{
|
|
||||||
k8s0-0 = { config, lib, pkgs, ... }:
|
|
||||||
let
|
|
||||||
base = baseConfig "k8s0-0";
|
|
||||||
etcd = etcdConfig "etcd0";
|
|
||||||
in
|
|
||||||
{
|
|
||||||
deployment.targetHost = "10.253.18.100";
|
|
||||||
require = [ base etcd kubeConfig kubeMaster kubeNode ];
|
|
||||||
services.dockerRegistry = {
|
|
||||||
enable = true;
|
|
||||||
listenAddress = "0.0.0.0";
|
|
||||||
extraConfig = {
|
|
||||||
REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver_cert}";
|
|
||||||
REGISTRY_HTTP_TLS_KEY = "${apiserver_key}";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
k8s0-1 = { config, lib, pkgs, ... }:
|
|
||||||
let
|
|
||||||
base = baseConfig "k8s0-1";
|
|
||||||
etcd = etcdConfig "etcd1";
|
|
||||||
in
|
|
||||||
{
|
|
||||||
deployment.targetHost = "10.253.18.101";
|
|
||||||
require = [ base etcd kubeConfig kubeNode ];
|
|
||||||
services.kubernetes.addons.dns.enable = false;
|
|
||||||
};
|
|
||||||
|
|
||||||
k8s0-2 = { config, lib, pkgs, ... }:
|
|
||||||
let
|
|
||||||
base = baseConfig "k8s0-2";
|
|
||||||
etcd = etcdConfig "etcd2";
|
|
||||||
in
|
|
||||||
{
|
|
||||||
deployment.targetHost = "10.253.18.102";
|
|
||||||
require = [ base etcd kubeConfig kubeNode ];
|
|
||||||
services.kubernetes.addons.dns.enable = false;
|
|
||||||
};
|
|
||||||
|
|
||||||
k8s0-3 = { config, lib, pkgs, ... }:
|
|
||||||
let
|
|
||||||
base = baseConfig "k8s0-3";
|
|
||||||
in
|
|
||||||
{
|
|
||||||
deployment.targetHost = "10.253.18.103";
|
|
||||||
require = [ base kubeConfig kubeNode ];
|
|
||||||
services.kubernetes.addons.dns.enable = false;
|
|
||||||
services.nfs.server = {
|
|
||||||
enable=true;
|
|
||||||
exports= ''
|
|
||||||
/vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
networking.firewall.allowedTCPPorts = [ 111 2049 ];
|
|
||||||
networking.firewall.allowedUDPPorts = [ 111 2049 ];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
173
lib/k8s.nix
Normal file
173
lib/k8s.nix
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
{ pkgs, kubeMaster, etcdNodes, clusterHosts, certs, ...}:
|
||||||
|
let
|
||||||
|
kubeApiserver = "https://${kubeMaster}:443";
|
||||||
|
localApiserver = "https://127.0.0.1:8080";
|
||||||
|
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
|
||||||
|
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
|
||||||
|
in
|
||||||
|
rec {
|
||||||
|
etcdConfig = name: {
|
||||||
|
services.etcd = {
|
||||||
|
inherit name;
|
||||||
|
enable = true;
|
||||||
|
listenClientUrls = ["https://0.0.0.0:2379"];
|
||||||
|
listenPeerUrls = ["https://0.0.0.0:2380"];
|
||||||
|
peerClientCertAuth = true;
|
||||||
|
keyFile = certs.etcd.key;
|
||||||
|
certFile = certs.etcd.cert;
|
||||||
|
trustedCaFile = certs.ca.cert;
|
||||||
|
advertiseClientUrls = [ "https://${name}:2379" ];
|
||||||
|
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
|
||||||
|
initialCluster = etcdCluster;
|
||||||
|
};
|
||||||
|
environment.variables = {
|
||||||
|
ETCDCTL_KEY_FILE = "${certs.admin.key}";
|
||||||
|
ETCDCTL_CERT_FILE = "${certs.admin.cert}";
|
||||||
|
ETCDCTL_CA_FILE = "${certs.ca.cert}";
|
||||||
|
ETCDCTL_PEERS = "https://127.0.0.1:2379";
|
||||||
|
};
|
||||||
|
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
|
||||||
|
systemd.services.flannel.after = [ "etcd.service" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
clientConf = instance: {
|
||||||
|
server = kubeApiserver;
|
||||||
|
keyFile = certs.${instance}.key;
|
||||||
|
certFile = certs.${instance}.cert;
|
||||||
|
caFile = certs.ca.cert;
|
||||||
|
};
|
||||||
|
|
||||||
|
kubeNode = instance: {
|
||||||
|
services.kubernetes = rec {
|
||||||
|
roles = [ "node" ];
|
||||||
|
kubeconfig = clientConf instance;
|
||||||
|
kubelet = {
|
||||||
|
enable = true;
|
||||||
|
clientCaFile = certs.ca.cert;
|
||||||
|
tlsKeyFile = certs.${instance}.key;
|
||||||
|
tlsCertFile = certs.${instance}.cert;
|
||||||
|
networkPlugin = null;
|
||||||
|
clusterDns = "10.0.0.254";
|
||||||
|
extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice";
|
||||||
|
inherit kubeconfig;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
networking = {
|
||||||
|
firewall = {
|
||||||
|
enable = true;
|
||||||
|
# trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
|
||||||
|
allowedTCPPorts = [ 53 4194 10250 ];
|
||||||
|
allowedUDPPorts = [ 53 ];
|
||||||
|
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
||||||
|
# systemd.services.kube-proxy.path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod];
|
||||||
|
};
|
||||||
|
|
||||||
|
kubeMaster = {
|
||||||
|
services.kubernetes = {
|
||||||
|
roles = [ "master" ];
|
||||||
|
kubelet.unschedulable = true;
|
||||||
|
apiserver = {
|
||||||
|
address = kubeMaster;
|
||||||
|
advertiseAddress = kubeMaster;
|
||||||
|
authorizationMode = [ "Node" "RBAC" ];
|
||||||
|
securePort = 443;
|
||||||
|
tlsKeyFile = certs.apiserver.key;
|
||||||
|
tlsCertFile = certs.apiserver.cert;
|
||||||
|
clientCaFile = certs.ca.cert;
|
||||||
|
kubeletClientCaFile = certs.ca.cert;
|
||||||
|
kubeletClientKeyFile = certs.apiserver.key;
|
||||||
|
kubeletClientCertFile = certs.apiserver.cert;
|
||||||
|
serviceAccountKeyFile = certs.apiserver.key;
|
||||||
|
};
|
||||||
|
scheduler.leaderElect = true;
|
||||||
|
controllerManager = {
|
||||||
|
leaderElect = true;
|
||||||
|
serviceAccountKeyFile = certs.apiserver.key;
|
||||||
|
rootCaFile = certs.ca.cert;
|
||||||
|
kubeconfig.server = localApiserver;
|
||||||
|
};
|
||||||
|
scheduler.kubeconfig.server = localApiserver;
|
||||||
|
addons.dashboard.enable = true;
|
||||||
|
addons.dns.enable = true;
|
||||||
|
};
|
||||||
|
networking.firewall = {
|
||||||
|
allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
|
||||||
|
# allowedUDPPorts = [ 4053 ];
|
||||||
|
};
|
||||||
|
environment.systemPackages = [ pkgs.kubernetes-helm ];
|
||||||
|
};
|
||||||
|
|
||||||
|
kubeConfig = instance: {
|
||||||
|
services.kubernetes = {
|
||||||
|
verbose = false;
|
||||||
|
caFile = certs.ca.cert;
|
||||||
|
flannel.enable = true;
|
||||||
|
clusterCidr = "10.10.0.0/16";
|
||||||
|
etcd = {
|
||||||
|
servers = etcdEndpoints;
|
||||||
|
keyFile = certs.apiserver.key;
|
||||||
|
certFile = certs.apiserver.cert;
|
||||||
|
caFile = certs.ca.cert;
|
||||||
|
};
|
||||||
|
proxy = {
|
||||||
|
kubeconfig = clientConf "kube-proxy";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
nixosConfig = node: {
|
||||||
|
imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ];
|
||||||
|
networking = {
|
||||||
|
hostName = node;
|
||||||
|
extraHosts = clusterHosts;
|
||||||
|
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
||||||
|
firewall.allowedTCPPorts = [ 80 443 ];
|
||||||
|
};
|
||||||
|
environment.systemPackages = [ pkgs.tshark ];
|
||||||
|
};
|
||||||
|
|
||||||
|
worker = host: ip: { config, lib, pkgs, ... }:
|
||||||
|
let
|
||||||
|
instance = host;
|
||||||
|
base = nixosConfig host;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
deployment.targetHost = ip;
|
||||||
|
require = [ base (kubeConfig instance) (kubeNode instance) ];
|
||||||
|
services.kubernetes.addons.dns.enable = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
server = host: etc: ip: { config, lib, pkgs, ... }:
|
||||||
|
let
|
||||||
|
instance = host;
|
||||||
|
base = nixosConfig instance;
|
||||||
|
etcd = etcdConfig etc;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
deployment.targetHost = ip;
|
||||||
|
require = [ base etcd (kubeConfig instance) (kubeNode instance) ];
|
||||||
|
services.kubernetes.addons.dns.enable = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
apiserver = host: ip: etc: { config, lib, pkgs, ... }:
|
||||||
|
let
|
||||||
|
instance = host;
|
||||||
|
base = nixosConfig instance;
|
||||||
|
etcd = etcdConfig etc;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
deployment.targetHost = ip;
|
||||||
|
require = [ base etcd (kubeConfig instance) kubeMaster (kubeNode instance) ];
|
||||||
|
services.dockerRegistry = {
|
||||||
|
enable = true;
|
||||||
|
listenAddress = "0.0.0.0";
|
||||||
|
extraConfig = {
|
||||||
|
REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}";
|
||||||
|
REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
124
lib/pki.nix
Normal file
124
lib/pki.nix
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
{ pkgs ? import <nixpkgs> {} }: rec {
|
||||||
|
ca-config = pkgs.writeText "ca-config.json" ''
|
||||||
|
{
|
||||||
|
"signing": {
|
||||||
|
"default": {
|
||||||
|
"expiry": "8760h"
|
||||||
|
},
|
||||||
|
"profiles": {
|
||||||
|
"kubernetes": {
|
||||||
|
"usages": [
|
||||||
|
"signing",
|
||||||
|
"key encipherment",
|
||||||
|
"server auth",
|
||||||
|
"client auth"
|
||||||
|
],
|
||||||
|
"expiry": "8760h"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
|
||||||
|
gencsr = args: pkgs.writeText "${args.name}-csr.json" ''
|
||||||
|
{
|
||||||
|
"CN": "${args.cn}",
|
||||||
|
"hosts": [ ${args.hosts} ],
|
||||||
|
"key": {
|
||||||
|
"algo": "rsa",
|
||||||
|
"size": 2048
|
||||||
|
},
|
||||||
|
"names": [
|
||||||
|
{
|
||||||
|
"O": "${args.o}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
|
||||||
|
initca =
|
||||||
|
let
|
||||||
|
ca_csr = gencsr {
|
||||||
|
name = "kubernetes";
|
||||||
|
cn = "kubernetes";
|
||||||
|
o = "kubernetes";
|
||||||
|
hosts = "";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
pkgs.runCommand "ca" {
|
||||||
|
buildInputs = [ pkgs.cfssl ];
|
||||||
|
} '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \
|
||||||
|
mkdir -p $out; cp *.pem $out'';
|
||||||
|
ca = {
|
||||||
|
key = "${initca}/ca-key.pem";
|
||||||
|
cert = "${initca}/ca.pem";
|
||||||
|
};
|
||||||
|
|
||||||
|
cfssl = conf: ''
|
||||||
|
cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
|
||||||
|
-config=${ca-config} -profile=kubernetes ${conf.csr} | \
|
||||||
|
cfssljson -bare cert; \
|
||||||
|
mkdir -p $out; cp *.pem $out
|
||||||
|
'';
|
||||||
|
|
||||||
|
gencert = conf:
|
||||||
|
let
|
||||||
|
drv = pkgs.runCommand "${conf.name}" {
|
||||||
|
buildInputs = [ pkgs.cfssl ];
|
||||||
|
} (cfssl conf);
|
||||||
|
in
|
||||||
|
{
|
||||||
|
key = "${drv}/cert-key.pem";
|
||||||
|
cert = "${drv}/cert.pem";
|
||||||
|
};
|
||||||
|
|
||||||
|
admin = gencert rec {
|
||||||
|
name = "admin";
|
||||||
|
csr = gencsr {
|
||||||
|
inherit name;
|
||||||
|
cn = "admin";
|
||||||
|
o = "system:masters";
|
||||||
|
hosts = "";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
apiserver = hosts:
|
||||||
|
gencert rec {
|
||||||
|
name = "kubernetes";
|
||||||
|
csr = gencsr {
|
||||||
|
inherit name hosts;
|
||||||
|
cn = "kubernetes";
|
||||||
|
o = "kubernetes";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
etcd = hosts: gencert rec {
|
||||||
|
name = "etcd";
|
||||||
|
csr = gencsr {
|
||||||
|
inherit name hosts;
|
||||||
|
cn = "etcd";
|
||||||
|
o = "kubernetes";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
kube-proxy = gencert rec {
|
||||||
|
name = "kube-proxy";
|
||||||
|
csr = gencsr {
|
||||||
|
inherit name;
|
||||||
|
cn = "system:kube-proxy";
|
||||||
|
o = "system:node-proxier";
|
||||||
|
hosts = "";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
worker = instance:
|
||||||
|
gencert rec {
|
||||||
|
name = instance.name;
|
||||||
|
csr = gencsr {
|
||||||
|
inherit name;
|
||||||
|
cn = "system:node:${instance.name}";
|
||||||
|
o = "system:nodes";
|
||||||
|
hosts = ''"${instance.name}","${instance.ip}"'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
{ config, pkgs, ... }:
|
{ config, pkgs, lib, ... }:
|
||||||
{
|
{
|
||||||
# Use the GRUB 2 boot loader.
|
# Use the GRUB 2 boot loader.
|
||||||
boot.loader.grub.enable = true;
|
boot.loader.grub.enable = true;
|
||||||
@@ -29,5 +29,16 @@
|
|||||||
|
|
||||||
security.rtkit.enable = true;
|
security.rtkit.enable = true;
|
||||||
|
|
||||||
imports = [ ./users.nix ./packages.nix ];
|
disabledModules = [ "services/cluster/kubernetes/default.nix" ];
|
||||||
|
|
||||||
|
imports = [
|
||||||
|
./users.nix
|
||||||
|
./packages.nix
|
||||||
|
./overlays/kubernetes.nix
|
||||||
|
];
|
||||||
|
|
||||||
|
nixpkgs.overlays = [
|
||||||
|
(import ./overlays/overlays.nix)
|
||||||
|
];
|
||||||
|
|
||||||
}
|
}
|
||||||
593
nixos/overlays/default.nix
Normal file
593
nixos/overlays/default.nix
Normal file
@@ -0,0 +1,593 @@
|
|||||||
|
{
|
||||||
|
callPackage,
|
||||||
|
coreutils,
|
||||||
|
docker,
|
||||||
|
e2fsprogs,
|
||||||
|
findutils,
|
||||||
|
go,
|
||||||
|
jshon,
|
||||||
|
jq,
|
||||||
|
lib,
|
||||||
|
pkgs,
|
||||||
|
pigz,
|
||||||
|
nixUnstable,
|
||||||
|
perl,
|
||||||
|
runCommand,
|
||||||
|
rsync,
|
||||||
|
shadow,
|
||||||
|
stdenv,
|
||||||
|
storeDir ? builtins.storeDir,
|
||||||
|
utillinux,
|
||||||
|
vmTools,
|
||||||
|
writeReferencesToFile,
|
||||||
|
writeScript,
|
||||||
|
writeText,
|
||||||
|
}:
|
||||||
|
|
||||||
|
# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
|
||||||
|
|
||||||
|
rec {
|
||||||
|
|
||||||
|
examples = import ./examples.nix {
|
||||||
|
inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb;
|
||||||
|
};
|
||||||
|
|
||||||
|
pullImage = callPackage ./pull.nix {};
|
||||||
|
|
||||||
|
# We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
|
||||||
|
# And we cannot untar it, because then we cannot preserve permissions ecc.
|
||||||
|
tarsum = runCommand "tarsum" {
|
||||||
|
buildInputs = [ go ];
|
||||||
|
} ''
|
||||||
|
mkdir tarsum
|
||||||
|
cd tarsum
|
||||||
|
|
||||||
|
cp ${./tarsum.go} tarsum.go
|
||||||
|
export GOPATH=$(pwd)
|
||||||
|
mkdir -p src/github.com/docker/docker/pkg
|
||||||
|
ln -sT ${docker.src}/components/engine/pkg/tarsum src/github.com/docker/docker/pkg/tarsum
|
||||||
|
go build
|
||||||
|
|
||||||
|
cp tarsum $out
|
||||||
|
'';
|
||||||
|
|
||||||
|
# buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
|
||||||
|
mergeDrvs = {
|
||||||
|
derivations,
|
||||||
|
onlyDeps ? false
|
||||||
|
}:
|
||||||
|
runCommand "merge-drvs" {
|
||||||
|
inherit derivations onlyDeps;
|
||||||
|
} ''
|
||||||
|
if [[ -n "$onlyDeps" ]]; then
|
||||||
|
echo $derivations > $out
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir $out
|
||||||
|
for derivation in $derivations; do
|
||||||
|
echo "Merging $derivation..."
|
||||||
|
if [[ -d "$derivation" ]]; then
|
||||||
|
# If it's a directory, copy all of its contents into $out.
|
||||||
|
cp -drf --preserve=mode -f $derivation/* $out/
|
||||||
|
else
|
||||||
|
# Otherwise treat the derivation as a tarball and extract it
|
||||||
|
# into $out.
|
||||||
|
tar -C $out -xpf $drv || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Helper for setting up the base files for managing users and
|
||||||
|
# groups, only if such files don't exist already. It is suitable for
|
||||||
|
# being used in a runAsRoot script.
|
||||||
|
shadowSetup = ''
|
||||||
|
export PATH=${shadow}/bin:$PATH
|
||||||
|
mkdir -p /etc/pam.d
|
||||||
|
if [[ ! -f /etc/passwd ]]; then
|
||||||
|
echo "root:x:0:0::/root:${stdenv.shell}" > /etc/passwd
|
||||||
|
echo "root:!x:::::::" > /etc/shadow
|
||||||
|
fi
|
||||||
|
if [[ ! -f /etc/group ]]; then
|
||||||
|
echo "root:x:0:" > /etc/group
|
||||||
|
echo "root:x::" > /etc/gshadow
|
||||||
|
fi
|
||||||
|
if [[ ! -f /etc/pam.d/other ]]; then
|
||||||
|
cat > /etc/pam.d/other <<EOF
|
||||||
|
account sufficient pam_unix.so
|
||||||
|
auth sufficient pam_rootok.so
|
||||||
|
password requisite pam_unix.so nullok sha512
|
||||||
|
session required pam_unix.so
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
if [[ ! -f /etc/login.defs ]]; then
|
||||||
|
touch /etc/login.defs
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Run commands in a virtual machine.
|
||||||
|
runWithOverlay = {
|
||||||
|
name,
|
||||||
|
fromImage ? null,
|
||||||
|
fromImageName ? null,
|
||||||
|
fromImageTag ? null,
|
||||||
|
diskSize ? 1024,
|
||||||
|
preMount ? "",
|
||||||
|
postMount ? "",
|
||||||
|
postUmount ? ""
|
||||||
|
}:
|
||||||
|
vmTools.runInLinuxVM (
|
||||||
|
runCommand name {
|
||||||
|
preVM = vmTools.createEmptyImage {
|
||||||
|
size = diskSize;
|
||||||
|
fullName = "docker-run-disk";
|
||||||
|
};
|
||||||
|
inherit fromImage fromImageName fromImageTag;
|
||||||
|
|
||||||
|
buildInputs = [ utillinux e2fsprogs jshon rsync ];
|
||||||
|
} ''
|
||||||
|
rm -rf $out
|
||||||
|
|
||||||
|
mkdir disk
|
||||||
|
mkfs /dev/${vmTools.hd}
|
||||||
|
mount /dev/${vmTools.hd} disk
|
||||||
|
cd disk
|
||||||
|
|
||||||
|
if [[ -n "$fromImage" ]]; then
|
||||||
|
echo "Unpacking base image..."
|
||||||
|
mkdir image
|
||||||
|
tar -C image -xpf "$fromImage"
|
||||||
|
|
||||||
|
# If the image name isn't set, read it from the image repository json.
|
||||||
|
if [[ -z "$fromImageName" ]]; then
|
||||||
|
fromImageName=$(jshon -k < image/repositories | head -n 1)
|
||||||
|
echo "From-image name wasn't set. Read $fromImageName."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# If the tag isn't set, use the name as an index into the json
|
||||||
|
# and read the first key found.
|
||||||
|
if [[ -z "$fromImageTag" ]]; then
|
||||||
|
fromImageTag=$(jshon -e $fromImageName -k < image/repositories \
|
||||||
|
| head -n1)
|
||||||
|
echo "From-image tag wasn't set. Read $fromImageTag."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use the name and tag to get the parent ID field.
|
||||||
|
parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
|
||||||
|
< image/repositories)
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Unpack all of the parent layers into the image.
|
||||||
|
lowerdir=""
|
||||||
|
while [[ -n "$parentID" ]]; do
|
||||||
|
echo "Unpacking layer $parentID"
|
||||||
|
mkdir -p image/$parentID/layer
|
||||||
|
tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
|
||||||
|
rm image/$parentID/layer.tar
|
||||||
|
|
||||||
|
find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
|
||||||
|
|
||||||
|
# Get the next lower directory and continue the loop.
|
||||||
|
lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
|
||||||
|
parentID=$(cat image/$parentID/json \
|
||||||
|
| (jshon -e parent -u 2>/dev/null || true))
|
||||||
|
done
|
||||||
|
|
||||||
|
mkdir work
|
||||||
|
mkdir layer
|
||||||
|
mkdir mnt
|
||||||
|
|
||||||
|
${lib.optionalString (preMount != "") ''
|
||||||
|
# Execute pre-mount steps
|
||||||
|
echo "Executing pre-mount steps..."
|
||||||
|
${preMount}
|
||||||
|
''}
|
||||||
|
|
||||||
|
if [ -n "$lowerdir" ]; then
|
||||||
|
mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
|
||||||
|
else
|
||||||
|
mount --bind layer mnt
|
||||||
|
fi
|
||||||
|
|
||||||
|
${lib.optionalString (postMount != "") ''
|
||||||
|
# Execute post-mount steps
|
||||||
|
echo "Executing post-mount steps..."
|
||||||
|
${postMount}
|
||||||
|
''}
|
||||||
|
|
||||||
|
umount mnt
|
||||||
|
|
||||||
|
(
|
||||||
|
cd layer
|
||||||
|
cmd='name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"'
|
||||||
|
find . -type c -exec bash -c "$cmd" \;
|
||||||
|
)
|
||||||
|
|
||||||
|
${postUmount}
|
||||||
|
'');
|
||||||
|
|
||||||
|
exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }:
|
||||||
|
runWithOverlay {
|
||||||
|
inherit name fromImage fromImageName fromImageTag diskSize;
|
||||||
|
|
||||||
|
postMount = ''
|
||||||
|
echo "Packing raw image..."
|
||||||
|
tar -C mnt --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out .
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
# Create an executable shell script which has the coreutils in its
|
||||||
|
# PATH. Since root scripts are executed in a blank environment, even
|
||||||
|
# things like `ls` or `echo` will be missing.
|
||||||
|
shellScript = name: text:
|
||||||
|
writeScript name ''
|
||||||
|
#!${stdenv.shell}
|
||||||
|
set -e
|
||||||
|
export PATH=${coreutils}/bin:/bin
|
||||||
|
${text}
|
||||||
|
'';
|
||||||
|
|
||||||
|
nixRegistration = contents: runCommand "nix-registration" {
|
||||||
|
buildInputs = [ nixUnstable perl ];
|
||||||
|
# For obtaining the closure of `contents'.
|
||||||
|
exportReferencesGraph =
|
||||||
|
let contentsList = if builtins.isList contents then contents else [ contents ];
|
||||||
|
in map (x: [("closure-" + baseNameOf x) x]) contentsList;
|
||||||
|
}
|
||||||
|
''
|
||||||
|
mkdir $out
|
||||||
|
printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out/db.dump
|
||||||
|
perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Create a "layer" (set of files).
|
||||||
|
mkPureLayer = {
|
||||||
|
# Name of the layer
|
||||||
|
name,
|
||||||
|
# JSON containing configuration and metadata for this layer.
|
||||||
|
baseJson,
|
||||||
|
# Files to add to the layer.
|
||||||
|
contents ? null,
|
||||||
|
# When copying the contents into the image, preserve symlinks to
|
||||||
|
# directories (see `rsync -K`). Otherwise, transform those symlinks
|
||||||
|
# into directories.
|
||||||
|
keepContentsDirlinks ? false,
|
||||||
|
# Additional commands to run on the layer before it is tar'd up.
|
||||||
|
extraCommands ? "", uid ? 0, gid ? 0
|
||||||
|
}:
|
||||||
|
runCommand "docker-layer-${name}" {
|
||||||
|
inherit baseJson contents extraCommands;
|
||||||
|
buildInputs = [ jshon rsync ];
|
||||||
|
}
|
||||||
|
''
|
||||||
|
mkdir layer
|
||||||
|
if [[ -n "$contents" ]]; then
|
||||||
|
echo "Adding contents..."
|
||||||
|
for item in $contents; do
|
||||||
|
echo "Adding $item"
|
||||||
|
rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
|
||||||
|
done
|
||||||
|
else
|
||||||
|
echo "No contents to add to layer."
|
||||||
|
fi
|
||||||
|
|
||||||
|
chmod ug+w layer
|
||||||
|
|
||||||
|
if [[ -n $extraCommands ]]; then
|
||||||
|
(cd layer; eval "$extraCommands")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Tar up the layer and throw it into 'layer.tar'.
|
||||||
|
echo "Packing layer..."
|
||||||
|
mkdir $out
|
||||||
|
tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
|
||||||
|
|
||||||
|
# Compute a checksum of the tarball.
|
||||||
|
echo "Computing layer checksum..."
|
||||||
|
tarsum=$(${tarsum} < $out/layer.tar)
|
||||||
|
|
||||||
|
# Add a 'checksum' field to the JSON, with the value set to the
|
||||||
|
# checksum of the tarball.
|
||||||
|
cat ${baseJson} | jshon -s "$tarsum" -i checksum > $out/json
|
||||||
|
|
||||||
|
# Indicate to docker that we're using schema version 1.0.
|
||||||
|
echo -n "1.0" > $out/VERSION
|
||||||
|
|
||||||
|
echo "Finished building layer '${name}'"
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Make a "root" layer; required if we need to execute commands as a
|
||||||
|
# privileged user on the image. The commands themselves will be
|
||||||
|
# performed in a virtual machine sandbox.
|
||||||
|
mkRootLayer = {
|
||||||
|
# Name of the image.
|
||||||
|
name,
|
||||||
|
# Script to run as root. Bash.
|
||||||
|
runAsRoot,
|
||||||
|
# Files to add to the layer. If null, an empty layer will be created.
|
||||||
|
contents ? null,
|
||||||
|
# When copying the contents into the image, preserve symlinks to
|
||||||
|
# directories (see `rsync -K`). Otherwise, transform those symlinks
|
||||||
|
# into directories.
|
||||||
|
keepContentsDirlinks ? false,
|
||||||
|
# JSON containing configuration and metadata for this layer.
|
||||||
|
baseJson,
|
||||||
|
# Existing image onto which to append the new layer.
|
||||||
|
fromImage ? null,
|
||||||
|
# Name of the image we're appending onto.
|
||||||
|
fromImageName ? null,
|
||||||
|
# Tag of the image we're appending onto.
|
||||||
|
fromImageTag ? null,
|
||||||
|
# How much disk to allocate for the temporary virtual machine.
|
||||||
|
diskSize ? 1024,
|
||||||
|
# Commands (bash) to run on the layer; these do not require sudo.
|
||||||
|
extraCommands ? ""
|
||||||
|
}:
|
||||||
|
# Generate an executable script from the `runAsRoot` text.
|
||||||
|
let runAsRootScript = shellScript "run-as-root.sh" runAsRoot;
|
||||||
|
in runWithOverlay {
|
||||||
|
name = "docker-layer-${name}";
|
||||||
|
|
||||||
|
inherit fromImage fromImageName fromImageTag diskSize;
|
||||||
|
|
||||||
|
preMount = lib.optionalString (contents != null && contents != []) ''
|
||||||
|
echo "Adding contents..."
|
||||||
|
for item in ${toString contents}; do
|
||||||
|
echo "Adding $item..."
|
||||||
|
rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
|
||||||
|
done
|
||||||
|
|
||||||
|
chmod ug+w layer
|
||||||
|
'';
|
||||||
|
|
||||||
|
postMount = ''
|
||||||
|
mkdir -p mnt/{dev,proc,sys} mnt${storeDir}
|
||||||
|
|
||||||
|
# Mount /dev, /sys and the nix store as shared folders.
|
||||||
|
mount --rbind /dev mnt/dev
|
||||||
|
mount --rbind /sys mnt/sys
|
||||||
|
mount --rbind ${storeDir} mnt${storeDir}
|
||||||
|
|
||||||
|
# Execute the run as root script. See 'man unshare' for
|
||||||
|
# details on what's going on here; basically this command
|
||||||
|
# means that the runAsRootScript will be executed in a nearly
|
||||||
|
# completely isolated environment.
|
||||||
|
unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript}
|
||||||
|
|
||||||
|
# Unmount directories and remove them.
|
||||||
|
umount -R mnt/dev mnt/sys mnt${storeDir}
|
||||||
|
rmdir --ignore-fail-on-non-empty \
|
||||||
|
mnt/dev mnt/proc mnt/sys mnt${storeDir} \
|
||||||
|
mnt$(dirname ${storeDir})
|
||||||
|
'';
|
||||||
|
|
||||||
|
postUmount = ''
|
||||||
|
(cd layer; eval "${extraCommands}")
|
||||||
|
|
||||||
|
echo "Packing layer..."
|
||||||
|
mkdir $out
|
||||||
|
tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar .
|
||||||
|
|
||||||
|
# Compute the tar checksum and add it to the output json.
|
||||||
|
echo "Computing checksum..."
|
||||||
|
ts=$(${tarsum} < $out/layer.tar)
|
||||||
|
cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
|
||||||
|
# Indicate to docker that we're using schema version 1.0.
|
||||||
|
echo -n "1.0" > $out/VERSION
|
||||||
|
|
||||||
|
echo "Finished building layer '${name}'"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# 1. extract the base image
|
||||||
|
# 2. create the layer
|
||||||
|
# 3. add layer deps to the layer itself, diffing with the base image
|
||||||
|
# 4. compute the layer id
|
||||||
|
# 5. put the layer in the image
|
||||||
|
# 6. repack the image
|
||||||
|
buildImage = args@{
|
||||||
|
# Image name.
|
||||||
|
name,
|
||||||
|
# Image tag.
|
||||||
|
tag ? "latest",
|
||||||
|
# Parent image, to append to.
|
||||||
|
fromImage ? null,
|
||||||
|
# Name of the parent image; will be read from the image otherwise.
|
||||||
|
fromImageName ? null,
|
||||||
|
# Tag of the parent image; will be read from the image otherwise.
|
||||||
|
fromImageTag ? null,
|
||||||
|
# Files to put on the image (a nix store path or list of paths).
|
||||||
|
contents ? null,
|
||||||
|
# When copying the contents into the image, preserve symlinks to
|
||||||
|
# directories (see `rsync -K`). Otherwise, transform those symlinks
|
||||||
|
# into directories.
|
||||||
|
keepContentsDirlinks ? false,
|
||||||
|
# Docker config; e.g. what command to run on the container.
|
||||||
|
config ? null,
|
||||||
|
# Optional bash script to run on the files prior to fixturizing the layer.
|
||||||
|
extraCommands ? "", uid ? 0, gid ? 0,
|
||||||
|
# Optional bash script to run as root on the image when provisioning.
|
||||||
|
runAsRoot ? null,
|
||||||
|
# Size of the virtual machine disk to provision when building the image.
|
||||||
|
diskSize ? 1024,
|
||||||
|
# Time of creation of the image.
|
||||||
|
created ? "1970-01-01T00:00:01Z",
|
||||||
|
}:
|
||||||
|
|
||||||
|
let
|
||||||
|
baseName = baseNameOf name;
|
||||||
|
|
||||||
|
# Create a JSON blob of the configuration. Set the date to unix zero.
|
||||||
|
baseJson = writeText "${baseName}-config.json" (builtins.toJSON {
|
||||||
|
inherit created config;
|
||||||
|
architecture = "amd64";
|
||||||
|
os = "linux";
|
||||||
|
});
|
||||||
|
|
||||||
|
layer =
|
||||||
|
if runAsRoot == null
|
||||||
|
then mkPureLayer {
|
||||||
|
name = baseName;
|
||||||
|
inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
|
||||||
|
} else mkRootLayer {
|
||||||
|
name = baseName;
|
||||||
|
inherit baseJson fromImage fromImageName fromImageTag
|
||||||
|
contents keepContentsDirlinks runAsRoot diskSize
|
||||||
|
extraCommands;
|
||||||
|
};
|
||||||
|
result = runCommand "docker-image-${baseName}.tar.gz" {
|
||||||
|
buildInputs = [ jshon pigz coreutils findutils jq ];
|
||||||
|
# Image name and tag must be lowercase
|
||||||
|
imageName = lib.toLower name;
|
||||||
|
imageTag = lib.toLower tag;
|
||||||
|
inherit fromImage baseJson;
|
||||||
|
layerClosure = writeReferencesToFile layer;
|
||||||
|
passthru.buildArgs = args;
|
||||||
|
passthru.layer = layer;
|
||||||
|
} ''
|
||||||
|
# Print tar contents:
|
||||||
|
# 1: Interpreted as relative to the root directory
|
||||||
|
# 2: With no trailing slashes on directories
|
||||||
|
# This is useful for ensuring that the output matches the
|
||||||
|
# values generated by the "find" command
|
||||||
|
ls_tar() {
|
||||||
|
for f in $(tar -tf $1 | xargs realpath -ms --relative-to=.); do
|
||||||
|
if [[ "$f" != "." ]]; then
|
||||||
|
echo "/$f"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdir image
|
||||||
|
touch baseFiles
|
||||||
|
if [[ -n "$fromImage" ]]; then
|
||||||
|
echo "Unpacking base image..."
|
||||||
|
tar -C image -xpf "$fromImage"
|
||||||
|
# Do not import the base image configuration and manifest
|
||||||
|
chmod a+w image image/*.json
|
||||||
|
rm -f image/*.json
|
||||||
|
|
||||||
|
if [[ -z "$fromImageName" ]]; then
|
||||||
|
fromImageName=$(jshon -k < image/repositories|head -n1)
|
||||||
|
fi
|
||||||
|
if [[ -z "$fromImageTag" ]]; then
|
||||||
|
fromImageTag=$(jshon -e $fromImageName -k \
|
||||||
|
< image/repositories|head -n1)
|
||||||
|
fi
|
||||||
|
parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
|
||||||
|
< image/repositories)
|
||||||
|
|
||||||
|
for l in image/*/layer.tar; do
|
||||||
|
ls_tar $l >> baseFiles
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
chmod -R ug+rw image
|
||||||
|
|
||||||
|
mkdir temp
|
||||||
|
cp ${layer}/* temp/
|
||||||
|
chmod ug+w temp/*
|
||||||
|
|
||||||
|
for dep in $(cat $layerClosure); do
|
||||||
|
find $dep >> layerFiles
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Adding layer..."
|
||||||
|
# Record the contents of the tarball with ls_tar.
|
||||||
|
ls_tar temp/layer.tar >> baseFiles
|
||||||
|
|
||||||
|
# Get the files in the new layer which were *not* present in
|
||||||
|
# the old layer, and record them as newFiles.
|
||||||
|
comm <(sort -n baseFiles|uniq) \
|
||||||
|
<(sort -n layerFiles|uniq|grep -v ${layer}) -1 -3 > newFiles
|
||||||
|
# Append the new files to the layer.
|
||||||
|
tar -rpf temp/layer.tar --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" \
|
||||||
|
--owner=0 --group=0 --no-recursion --files-from newFiles
|
||||||
|
|
||||||
|
echo "Adding meta..."
|
||||||
|
|
||||||
|
# If we have a parentID, add it to the json metadata.
|
||||||
|
if [[ -n "$parentID" ]]; then
|
||||||
|
cat temp/json | jshon -s "$parentID" -i parent > tmpjson
|
||||||
|
mv tmpjson temp/json
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Take the sha256 sum of the generated json and use it as the layer ID.
|
||||||
|
# Compute the size and add it to the json under the 'Size' field.
|
||||||
|
layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
|
||||||
|
size=$(stat --printf="%s" temp/layer.tar)
|
||||||
|
cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
|
||||||
|
mv tmpjson temp/json
|
||||||
|
|
||||||
|
# Use the temp folder we've been working on to create a new image.
|
||||||
|
mv temp image/$layerID
|
||||||
|
|
||||||
|
# Create image json and image manifest
|
||||||
|
imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
|
||||||
|
manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
|
||||||
|
currentID=$layerID
|
||||||
|
while [[ -n "$currentID" ]]; do
|
||||||
|
layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
|
||||||
|
imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
|
||||||
|
imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
|
||||||
|
manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
|
||||||
|
|
||||||
|
currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
|
||||||
|
done
|
||||||
|
|
||||||
|
imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
|
||||||
|
echo "$imageJson" > "image/$imageJsonChecksum.json"
|
||||||
|
manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
|
||||||
|
echo "$manifestJson" > image/manifest.json
|
||||||
|
|
||||||
|
# Store the json under the name image/repositories.
|
||||||
|
jshon -n object \
|
||||||
|
-n object -s "$layerID" -i "$imageTag" \
|
||||||
|
-i "$imageName" > image/repositories
|
||||||
|
|
||||||
|
# Make the image read-only.
|
||||||
|
chmod -R a-w image
|
||||||
|
|
||||||
|
echo "Cooking the image..."
|
||||||
|
tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out
|
||||||
|
|
||||||
|
echo "Finished."
|
||||||
|
'';
|
||||||
|
|
||||||
|
in
|
||||||
|
result;
|
||||||
|
|
||||||
|
# Build an image and populate its nix database with the provided
|
||||||
|
# contents. The main purpose is to be able to use nix commands in
|
||||||
|
# the container.
|
||||||
|
# Be careful since this doesn't work well with multilayer.
|
||||||
|
buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }:
|
||||||
|
buildImage (args // {
|
||||||
|
extraCommands = ''
|
||||||
|
echo "Generating the nix database..."
|
||||||
|
echo "Warning: only the database of the deepest Nix layer is loaded."
|
||||||
|
echo " If you want to use nix commands in the container, it would"
|
||||||
|
echo " be better to only have one layer that contains a nix store."
|
||||||
|
# This requires Nix 1.12 or higher
|
||||||
|
export NIX_REMOTE=local?root=$PWD
|
||||||
|
${nixUnstable}/bin/nix-store --load-db < ${nixRegistration contents}/db.dump
|
||||||
|
|
||||||
|
# We fill the store in order to run the 'verify' command that
|
||||||
|
# generates hash and size of output paths.
|
||||||
|
# Note when Nix 1.12 is be the stable one, the database dump
|
||||||
|
# generated by the exportReferencesGraph function will
|
||||||
|
# contains sha and size. See
|
||||||
|
# https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a
|
||||||
|
storePaths=$(cat ${nixRegistration contents}/storePaths)
|
||||||
|
echo "Copying everything to /nix/store (will take a while)..."
|
||||||
|
cp -prd $storePaths nix/store/
|
||||||
|
${nixUnstable}/bin/nix-store --verify --check-contents
|
||||||
|
|
||||||
|
mkdir -p nix/var/nix/gcroots/docker/
|
||||||
|
for i in ${lib.concatStringsSep " " contents}; do
|
||||||
|
ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
|
||||||
|
done;
|
||||||
|
'' + extraCommands;
|
||||||
|
});
|
||||||
|
}
|
||||||
40
nixos/overlays/detjson.py
Normal file
40
nixos/overlays/detjson.py
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Deterministic layer json: https://github.com/docker/hub-feedback/issues/488
|
||||||
|
|
||||||
|
import sys
|
||||||
|
reload(sys)
|
||||||
|
sys.setdefaultencoding('UTF8')
|
||||||
|
import json
|
||||||
|
|
||||||
|
# If any of the keys below are equal to a certain value
|
||||||
|
# then we can delete it because it's the default value
|
||||||
|
SAFEDELS = {
|
||||||
|
"Size": 0,
|
||||||
|
"config": {
|
||||||
|
"ExposedPorts": None,
|
||||||
|
"MacAddress": "",
|
||||||
|
"NetworkDisabled": False,
|
||||||
|
"PortSpecs": None,
|
||||||
|
"VolumeDriver": ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SAFEDELS["container_config"] = SAFEDELS["config"]
|
||||||
|
|
||||||
|
def makedet(j, safedels):
|
||||||
|
for k,v in safedels.items():
|
||||||
|
if k not in j:
|
||||||
|
continue
|
||||||
|
if type(v) == dict:
|
||||||
|
makedet(j[k], v)
|
||||||
|
elif j[k] == v:
|
||||||
|
del j[k]
|
||||||
|
|
||||||
|
def main():
|
||||||
|
j = json.load(sys.stdin)
|
||||||
|
makedet(j, SAFEDELS)
|
||||||
|
json.dump(j, sys.stdout, sort_keys=True)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
127
nixos/overlays/examples.nix
Normal file
127
nixos/overlays/examples.nix
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
# Examples of using the docker tools to build packages.
|
||||||
|
#
|
||||||
|
# This file defines several docker images. In order to use an image,
|
||||||
|
# build its derivation with `nix-build`, and then load the result with
|
||||||
|
# `docker load`. For example:
|
||||||
|
#
|
||||||
|
# $ nix-build '<nixpkgs>' -A dockerTools.examples.redis
|
||||||
|
# $ docker load < result
|
||||||
|
|
||||||
|
{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }:
|
||||||
|
|
||||||
|
rec {
|
||||||
|
# 1. basic example
|
||||||
|
bash = buildImage {
|
||||||
|
name = "bash";
|
||||||
|
contents = pkgs.bashInteractive;
|
||||||
|
};
|
||||||
|
|
||||||
|
# 2. service example, layered on another image
|
||||||
|
redis = buildImage {
|
||||||
|
name = "redis";
|
||||||
|
tag = "latest";
|
||||||
|
|
||||||
|
# for example's sake, we can layer redis on top of bash or debian
|
||||||
|
fromImage = bash;
|
||||||
|
# fromImage = debian;
|
||||||
|
|
||||||
|
contents = pkgs.redis;
|
||||||
|
runAsRoot = ''
|
||||||
|
mkdir -p /data
|
||||||
|
'';
|
||||||
|
|
||||||
|
config = {
|
||||||
|
Cmd = [ "/bin/redis-server" ];
|
||||||
|
WorkingDir = "/data";
|
||||||
|
Volumes = {
|
||||||
|
"/data" = {};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# 3. another service example
|
||||||
|
nginx = let
|
||||||
|
nginxPort = "80";
|
||||||
|
nginxConf = pkgs.writeText "nginx.conf" ''
|
||||||
|
user nginx nginx;
|
||||||
|
daemon off;
|
||||||
|
error_log /dev/stdout info;
|
||||||
|
pid /dev/null;
|
||||||
|
events {}
|
||||||
|
http {
|
||||||
|
access_log /dev/stdout;
|
||||||
|
server {
|
||||||
|
listen ${nginxPort};
|
||||||
|
index index.html;
|
||||||
|
location / {
|
||||||
|
root ${nginxWebRoot};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
nginxWebRoot = pkgs.writeTextDir "index.html" ''
|
||||||
|
<html><body><h1>Hello from NGINX</h1></body></html>
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
buildImage {
|
||||||
|
name = "nginx-container";
|
||||||
|
contents = pkgs.nginx;
|
||||||
|
|
||||||
|
runAsRoot = ''
|
||||||
|
#!${pkgs.stdenv.shell}
|
||||||
|
${shadowSetup}
|
||||||
|
groupadd --system nginx
|
||||||
|
useradd --system --gid nginx nginx
|
||||||
|
'';
|
||||||
|
|
||||||
|
config = {
|
||||||
|
Cmd = [ "nginx" "-c" nginxConf ];
|
||||||
|
ExposedPorts = {
|
||||||
|
"${nginxPort}/tcp" = {};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# 4. example of pulling an image. could be used as a base for other images
|
||||||
|
nixFromDockerHub = pullImage {
|
||||||
|
imageName = "nixos/nix";
|
||||||
|
imageTag = "1.11";
|
||||||
|
# this hash will need change if the tag is updated at docker hub
|
||||||
|
sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83";
|
||||||
|
};
|
||||||
|
|
||||||
|
# 5. example of multiple contents, emacs and vi happily coexisting
|
||||||
|
editors = buildImage {
|
||||||
|
name = "editors";
|
||||||
|
contents = [
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.bash
|
||||||
|
pkgs.emacs
|
||||||
|
pkgs.vim
|
||||||
|
pkgs.nano
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
# 6. nix example to play with the container nix store
|
||||||
|
# docker run -it --rm nix nix-store -qR $(nix-build '<nixpkgs>' -A nix)
|
||||||
|
nix = buildImageWithNixDb {
|
||||||
|
name = "nix";
|
||||||
|
contents = [
|
||||||
|
# nix-store uses cat program to display results as specified by
|
||||||
|
# the image env variable NIX_PAGER.
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.nix
|
||||||
|
];
|
||||||
|
config = {
|
||||||
|
Env = [ "NIX_PAGER=cat" ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# 7. example of adding something on top of an image pull by our
|
||||||
|
# dockerTools chain.
|
||||||
|
onTopOfPulledImage = buildImage {
|
||||||
|
name = "onTopOfPulledImage";
|
||||||
|
fromImage = nixFromDockerHub;
|
||||||
|
contents = [ pkgs.hello ];
|
||||||
|
};
|
||||||
|
}
|
||||||
1157
nixos/overlays/kubernetes.nix
Normal file
1157
nixos/overlays/kubernetes.nix
Normal file
File diff suppressed because it is too large
Load Diff
4
nixos/overlays/overlays.nix
Normal file
4
nixos/overlays/overlays.nix
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
self: super: {
|
||||||
|
dockerTools = super.callPackage ./overlay/default.nix { go = self.go_1_9; };
|
||||||
|
# super.config.services.kubernetes = super.callPackage ./overlay/kubernetes.nix {};
|
||||||
|
}
|
||||||
32
nixos/overlays/pull.nix
Normal file
32
nixos/overlays/pull.nix
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
|
||||||
|
let
|
||||||
|
nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
|
||||||
|
in
|
||||||
|
# For simplicity we only support sha256.
|
||||||
|
{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
|
||||||
|
, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
|
||||||
|
let
|
||||||
|
pullImage = vmTools.runInLinuxVM (
|
||||||
|
stdenv.mkDerivation {
|
||||||
|
inherit name imageId;
|
||||||
|
|
||||||
|
certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
|
||||||
|
|
||||||
|
builder = ./pull.sh;
|
||||||
|
|
||||||
|
nativeBuildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
|
||||||
|
|
||||||
|
outputHashAlgo = "sha256";
|
||||||
|
outputHash = sha256;
|
||||||
|
|
||||||
|
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
|
||||||
|
|
||||||
|
preVM = vmTools.createEmptyImage {
|
||||||
|
size = 2048;
|
||||||
|
fullName = "${name}-disk";
|
||||||
|
};
|
||||||
|
|
||||||
|
QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
|
||||||
|
});
|
||||||
|
in
|
||||||
|
pullImage
|
||||||
36
nixos/overlays/pull.sh
Normal file
36
nixos/overlays/pull.sh
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
source $stdenv/setup
|
||||||
|
|
||||||
|
mkdir -p /var/lib/docker
|
||||||
|
mkfs.ext4 /dev/vda
|
||||||
|
mount -t ext4 /dev/vda /var/lib/docker
|
||||||
|
|
||||||
|
modprobe virtio_net
|
||||||
|
dhclient eth0
|
||||||
|
|
||||||
|
mkdir -p /etc/ssl/certs/
|
||||||
|
cp "$certs" "/etc/ssl/certs/"
|
||||||
|
|
||||||
|
# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
|
||||||
|
mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
|
||||||
|
cd /sys/fs/cgroup
|
||||||
|
for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
|
||||||
|
mkdir -p $sys
|
||||||
|
if ! mountpoint -q $sys; then
|
||||||
|
if ! mount -n -t cgroup -o $sys cgroup $sys; then
|
||||||
|
rmdir $sys || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# run docker daemon
|
||||||
|
dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
|
||||||
|
|
||||||
|
until docker ps 2>/dev/null; do
|
||||||
|
printf '.'
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
rm -r $out
|
||||||
|
|
||||||
|
docker pull ${imageId}
|
||||||
|
docker save ${imageId} > $out
|
||||||
24
nixos/overlays/tarsum.go
Normal file
24
nixos/overlays/tarsum.go
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"github.com/docker/docker/pkg/tarsum"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ts, err := tarsum.NewTarSum(os.Stdin, true, tarsum.Version1)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err = io.Copy(ioutil.Discard, ts); err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(ts.Sum(nil))
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user