fix: Add cilium config for Rossby
This commit is contained in:
@@ -1,32 +0,0 @@
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
environment.systemPackages = with pkgs; [ cilium-cli ];
|
||||
services.kubernetes.kubelet.cni.packages = with pkgs; [ cni-plugin-cilium ];
|
||||
services.kubernetes.kubelet.cni.config = [
|
||||
{
|
||||
name = "cilium";
|
||||
type = "cilium-cni";
|
||||
cniVersion = "0.3.1";
|
||||
}
|
||||
];
|
||||
networking = {
|
||||
firewall = {
|
||||
allowPing = true;
|
||||
logReversePathDrops = true;
|
||||
checkReversePath = false;
|
||||
allowedTCPPorts = [
|
||||
4240 # clilum healthcheck
|
||||
4244 # hubble api
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
8472 # clilum vxlan
|
||||
];
|
||||
};
|
||||
};
|
||||
boot.kernelModules = [
|
||||
"ip6_tables"
|
||||
"ip6table_mangle"
|
||||
"ip6table_raw"
|
||||
"ip6table_filter"
|
||||
];
|
||||
}
|
||||
@@ -213,6 +213,14 @@ let
|
||||
pki.caCertPathPrefix = "${pki.initca}/ca";
|
||||
|
||||
kubelet = {
|
||||
cni.packages = [ pkgs.cni-plugin-cilium ];
|
||||
cni.config = [
|
||||
{
|
||||
name = "cilium";
|
||||
type = "cilium-cni";
|
||||
cniVersion = "0.3.1";
|
||||
}
|
||||
];
|
||||
# clusterDomain = "${cfg.clusterName}.local";
|
||||
};
|
||||
|
||||
@@ -222,7 +230,7 @@ let
|
||||
allowPrivileged = true;
|
||||
securePort = 6443;
|
||||
serviceClusterIpRange = "10.0.0.0/22";
|
||||
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
|
||||
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert} --allow-privileged true";
|
||||
#extraSANs = cfg.master.extraSANs;
|
||||
verbosity = 2;
|
||||
etcd.servers =
|
||||
@@ -252,13 +260,31 @@ let
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [ 53 5000 8080 6443 4001 2379 2380 10250 10251 10252 ];
|
||||
allowedUDPPorts = [ 53 4053 ];
|
||||
allowedTCPPorts = [
|
||||
53
|
||||
5000
|
||||
8080
|
||||
6443
|
||||
4001
|
||||
2379
|
||||
2380
|
||||
10250
|
||||
10251
|
||||
10252
|
||||
4240 # clilum healthcheck
|
||||
4244 # hubble api
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
53
|
||||
4053
|
||||
8472 # clilum vxlan
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.kubernetes-helm
|
||||
pkgs.kubectl
|
||||
pkgs.cilium-cli
|
||||
cluster-scripts
|
||||
];
|
||||
|
||||
@@ -337,13 +363,23 @@ let
|
||||
};
|
||||
|
||||
kubeNode = {
|
||||
services.kubernetes = rec {
|
||||
services.kubernetes = {
|
||||
roles = [ "node" ];
|
||||
inherit apiserverAddress;
|
||||
# masterAddress = cfg.master.name;
|
||||
masterAddress = "${cfg.master.name}.${domain}";
|
||||
clusterCidr = cfg.cidr;
|
||||
kubelet.cni.packages = [ pkgs.cni-plugin-cilium ];
|
||||
kubelet.cni.config = [
|
||||
{
|
||||
name = "cilium";
|
||||
type = "cilium-cni";
|
||||
cniVersion = "0.3.1";
|
||||
}
|
||||
];
|
||||
# kubelet.clusterDomain = "${cfg.clusterName}.local";
|
||||
# Cilium replaces kube-proxy
|
||||
kube-proxy.enable = false;
|
||||
kubelet.hostname = "${hostName}";
|
||||
proxy.hostname = "${hostName}";
|
||||
proxy.extraOpts = "--metrics-bind-address 0.0.0.0:10249";
|
||||
@@ -352,12 +388,35 @@ let
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [ 4194 10250 ];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
allowedTCPPorts = [
|
||||
4194
|
||||
10250
|
||||
4240 # clilum healthcheck
|
||||
4244 # hubble api
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
53
|
||||
8472 # clilum vxlan
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
virtualisation.docker.enable = false; # conflicts with containerd!
|
||||
virtualisation.docker.autoPrune.enable = pkgs.lib.mkForce false; # conflicts with linkerd2
|
||||
virtualisation.containerd.enable = true;
|
||||
# required to get it to pick up cilium-cni as placed by the agent
|
||||
virtualisation.containerd.settings = {
|
||||
# version = lib.mkForce 3; # TODO: unclear if we should do this
|
||||
metrics = {
|
||||
address = "[::]:9103";
|
||||
};
|
||||
# Was being ignored as unknown?
|
||||
# Must set otherwise the module sets it to the Nix store, which Cilium can't write to
|
||||
plugins."io.containerd.grpc.v1.cri".cni = {
|
||||
bin_dir = "/opt/cni/bin";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
||||
description = "Kubernetes certmgr bootstrapper";
|
||||
wantedBy = [ "certmgr.service" ];
|
||||
@@ -470,4 +529,3 @@ in {
|
||||
../overrides/kubelet.nix
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
self: super:
|
||||
let
|
||||
lib = super.lib;
|
||||
buildGoModule = super.buildGoModule;
|
||||
fetchFromGitHub = super.fetchFromGitHub;
|
||||
in
|
||||
{
|
||||
cni-plugin-cilium = buildGoModule rec {
|
||||
pname = "cilium-cni";
|
||||
version = "1.17.3";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "cilium";
|
||||
repo = "cilium";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-HcKRenRILpJCzJZbOYzrQrLlEeif9J9jJDKFzA6NtXc=";
|
||||
};
|
||||
|
||||
vendorHash = null;
|
||||
|
||||
ldflags = [
|
||||
"-s"
|
||||
"-w"
|
||||
"-X main.Version=${version}"
|
||||
"-X main.Commit=${version}"
|
||||
"-X main.Program=cilium"
|
||||
];
|
||||
|
||||
subPackages = [ "./plugins/cilium-cni" ];
|
||||
|
||||
# level=error msg="unable to open \"/sys/devices/system/cpu/possible\"" error="open /sys/devices/system/cpu/possible: no such file or directory" subsys=datapath-loader
|
||||
doCheck = false;
|
||||
|
||||
meta = with lib; {
|
||||
description = "Cilium CNI plugin";
|
||||
homepage = "https://github.com/cilium/cilium/";
|
||||
license = licenses.asl20;
|
||||
platforms = platforms.linux;
|
||||
maintainers = with maintainers; [ starcraft66 ];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -147,7 +147,7 @@ in
|
||||
|
||||
extraSANs = mkOption {
|
||||
description = "Extra x509 Subject Alternative Names to be added to the kubelet tls cert.";
|
||||
default = [];
|
||||
default = [ ];
|
||||
type = listOf str;
|
||||
};
|
||||
|
||||
|
||||
2
nixos
2
nixos
Submodule nixos updated: 444d8f8008...36d9c7e052
@@ -1,4 +1,7 @@
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
...
|
||||
}:
|
||||
let
|
||||
computeNodes = import ../c0/nodes.nix ++ [
|
||||
rec {
|
||||
|
||||
@@ -10,22 +10,53 @@ let
|
||||
cryptsetup = super.cryptsetup;
|
||||
|
||||
singularity = super.singularity.overrideAttrs (attrs: rec {
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
make -C builddir install LOCALSTATEDIR=$out/var
|
||||
chmod 755 $out/libexec/singularity/bin/starter-suid
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
make -C builddir install LOCALSTATEDIR=$out/var
|
||||
chmod 755 $out/libexec/singularity/bin/starter-suid
|
||||
|
||||
# Explicitly configure paths in the config file
|
||||
sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf
|
||||
sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf
|
||||
sed -i 's|^shared loop devices = no|shared loop devices = yes|' $out/etc/singularity/singularity.conf
|
||||
# Explicitly configure paths in the config file
|
||||
sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${
|
||||
lib.makeBinPath [ squashfsTools ]
|
||||
}/mksquashfs|' $out/etc/singularity/singularity.conf
|
||||
sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [ cryptsetup ]}/cryptsetup|' $out/etc/singularity/singularity.conf
|
||||
sed -i 's|^shared loop devices = no|shared loop devices = yes|' $out/etc/singularity/singularity.conf
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
runHook postInstall
|
||||
'';
|
||||
});
|
||||
|
||||
cni-plugin-cilium = super.buildGoModule rec {
|
||||
pname = "cilium-cni";
|
||||
version = "1.16.2";
|
||||
|
||||
src = super.fetchFromGitHub {
|
||||
owner = "cilium";
|
||||
repo = "cilium";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-HcKRenRILpJCzJZbOYzrQrLlEeif9J9jJDKFzA6NtXc=";
|
||||
};
|
||||
|
||||
vendorHash = null;
|
||||
|
||||
ldflags = [
|
||||
"-s"
|
||||
"-w"
|
||||
"-X main.Version=${version}"
|
||||
"-X main.Commit=${version}"
|
||||
"-X main.Program=cilium"
|
||||
];
|
||||
|
||||
subPackages = [ "./plugins/cilium-cni" ];
|
||||
|
||||
# level=error msg="unable to open \"/sys/devices/system/cpu/possible\"" error="open /sys/devices/system/cpu/possible: no such file or directory" subsys=datapath-loader
|
||||
doCheck = false;
|
||||
# TODO: Maybe not needed
|
||||
doInstallCheck = true;
|
||||
};
|
||||
in
|
||||
{
|
||||
inherit singularity;
|
||||
# inherit msmtp;
|
||||
inherit singularity;
|
||||
inherit cni-plugin-cilium;
|
||||
# inherit msmtp;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user