From 899a7f4338a8c099293746a2e0266d282d7b5cf3 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Sat, 6 Sep 2025 08:01:54 +0200 Subject: [PATCH] fix: misc fixes (save for rossby) --- LICENSE | 2 +- cluster/c0/default.nix | 2 +- cluster/c0/kernel.nix | 10 + cluster/c1/default.nix | 4 +- cluster/cluster.nix | 16 +- cluster/ekman/default.nix | 20 +- cluster/fs-work/default.nix | 2 +- cluster/mounts.nix | 14 +- cluster/users.nix | 103 ++++++- configuration.nix | 19 +- modules/hpc/hpc.nix | 13 +- modules/k8s/default.nix | 6 +- modules/overrides/kubelet.nix | 5 +- modules/overrides/kubernetes_default.nix2 | 358 ++++++++++++++++++++++ 14 files changed, 535 insertions(+), 39 deletions(-) create mode 100644 modules/overrides/kubernetes_default.nix2 diff --git a/LICENSE b/LICENSE index 6c15032..54476bf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Jonas Juselius, Serit IT Partner Tromsø +Copyright (c) 2025 Jonas Juselius, Oceanbox AS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/cluster/c0/default.nix b/cluster/c0/default.nix index 1d76fd2..0a103fb 100644 --- a/cluster/c0/default.nix +++ b/cluster/c0/default.nix @@ -99,7 +99,7 @@ let hw ../cluster.nix ../mounts.nix - # ./kernel.nix + #./kernel.nix ]; } // compute; diff --git a/cluster/c0/kernel.nix b/cluster/c0/kernel.nix index 17b55e4..6aaa81c 100644 --- a/cluster/c0/kernel.nix +++ b/cluster/c0/kernel.nix @@ -40,6 +40,16 @@ let in { # i40e2 = i40e; + # boot.kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override { + # argsOverride = rec { + # src = pkgs.fetchurl { + # url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz"; + # sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid"; + # }; + # version = "5.10.239"; + # modDirVersion = "5.10.239"; + # }; + # }); boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10; # overlay = self: super: { # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; diff --git a/cluster/c1/default.nix b/cluster/c1/default.nix index aac978e..c5d1597 100644 --- a/cluster/c1/default.nix +++ b/cluster/c1/default.nix @@ -45,7 +45,7 @@ let name = host.name; address = host.address; }; - os.externalInterface = "eno33"; + os.externalInterface = "eno33np0"; hpc.compute = true; # k8s = { inherit etcdCluster; }; }; @@ -74,7 +74,7 @@ let networking = { hostName = host.name; useDHCP = false; - interfaces.eno33 = { + interfaces.eno33np0 = { useDHCP = false; ipv4.addresses = [ { address = host.address; diff --git a/cluster/cluster.nix b/cluster/cluster.nix index 8db5564..7f821d0 100644 --- a/cluster/cluster.nix +++ b/cluster/cluster.nix @@ -88,9 +88,9 @@ let "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5k0dXn60dZ3iORy99LVvgTldu9nYU1TJVL1wCJEqp kaih kubernetes" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHVwcJOtx9YTWy+aD4xGbyPFLOdMN6NqY8wcfDtHczyT Stig Rune Jensen" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfgY468dPNpdXZCkD9jw1p2qA0+z56Wi/c1VYE+riki Stig Rune Jensen" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst" ]; docker.enable = false; }; @@ -202,9 +202,9 @@ let master = { name = "frontend"; address = "10.255.241.99"; - extraSANs = [ - "frontend.oceanbox.io" - ]; + # extraSANs = [ + # "frontend.oceanbox.io" + # ]; }; ingressNodes = [ "ekman.oceanbox.io" @@ -222,10 +222,10 @@ let copyCaKey.text = "cp ${./ca}/ca-key.pem /var/lib/kubernetes/secrets"; }; - services.kubernetes.kubelet.extraSANs = mkSANs { - name = cfg.name; - address = cfg.address; - }; + # services.kubernetes.kubelet.extraSANs = mkSANs { + # name = cfg.name; + # address = cfg.address; + # }; }; shosts = { diff --git a/cluster/ekman/default.nix b/cluster/ekman/default.nix index e2dd11b..a938248 100644 --- a/cluster/ekman/default.nix +++ b/cluster/ekman/default.nix @@ -23,7 +23,7 @@ in rdma.enable = true; automount.enable = true; home = false; - opt = true; + opt = false; work = true; data = true; backup = true; @@ -197,6 +197,22 @@ in device = "/home"; options = [ "bind" ]; }; + "/exports/opt/bin" = { + device = "/opt/bin"; + options = [ "bind" ]; + }; + "/exports/opt/sif" = { + device = "/opt/sif"; + options = [ "bind" ]; + }; + "/exports/opt/singularity" = { + device = "/opt/singularity"; + options = [ "bind" ]; + }; + "/exports/nfs-provisioner" = { + device = "/vol/nfs-provisioner"; + options = [ "bind" ]; + }; "/frontend" = { device = "/home"; options = [ "bind" ]; @@ -298,6 +314,8 @@ in chmod 755 /home/olean chmod 755 /home/frankgaa chmod 755 /home/jonas + chmod 755 /home/mrtz + chmod 755 /home/avle chmod 755 /home/stig chmod 755 /home/bast chmod 755 /home/simenlk diff --git a/cluster/fs-work/default.nix b/cluster/fs-work/default.nix index dc28a41..5db73d7 100644 --- a/cluster/fs-work/default.nix +++ b/cluster/fs-work/default.nix @@ -130,7 +130,7 @@ in { # interfaces."ibp65s0.7666" = { # useDHCP = false; # }; - interfaces."ibp1s0f0" = { + interfaces.ibp1s0f0 = { useDHCP = false; ipv4.addresses = [ { diff --git a/cluster/mounts.nix b/cluster/mounts.nix index e86f9b2..180a1bd 100644 --- a/cluster/mounts.nix +++ b/cluster/mounts.nix @@ -10,7 +10,6 @@ let options = [ "soft" "defaults" "vers=4.2" ] ++ - (if cfg.rdma.enable then [ "rdma" ] else []) ++ (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []); home = @@ -28,19 +27,22 @@ let } else {}; opt = + let + server = "10.255.241.100"; + in if cfg.opt then { "/opt/bin" = { - device = "10.255.${subnet}.90:/opt/bin"; + device = "${server}:/opt/bin"; fsType = "nfs4"; inherit options; }; "/opt/sif" = { - device = "10.255.${subnet}.90:/opt/sif"; + device = "${server}:/opt/sif"; fsType = "nfs4"; inherit options; }; "/opt/singularity" = { - device = "10.255.${subnet}.90:/opt/singularity"; + device = "${server}:/opt/singularity"; fsType = "nfs4"; inherit options; }; @@ -69,7 +71,7 @@ let "/work" = { device = "10.255.${subnet}.90:/work"; fsType = "nfs4"; - inherit options; + options = options ++ (if cfg.rdma.enable then [ "rdma" ] else []); }; } else {}; @@ -78,7 +80,7 @@ let "/backup" = { device = "10.255.${subnet}.80:/backup"; fsType = "nfs4"; - options = options ++ [ "ro" ]; + options = options ++ [ "ro" ] ++ (if cfg.rdma.enable then [ "rdma" ] else []); }; } else {}; diff --git a/cluster/users.nix b/cluster/users.nix index 955f76e..c9c4259 100644 --- a/cluster/users.nix +++ b/cluster/users.nix @@ -9,7 +9,11 @@ simenlk = { gid = 1005; }; isa = { gid = 1006; }; ole = { gid = 1007; }; - moritz = { gid = 1008; }; + mrtz = { gid = 1008; }; + avle = { gid = 1009; }; + lilly = { gid = 1010; }; + + # kaihc = { gid = 3001; }; hipster = { members = [ @@ -18,6 +22,7 @@ "frankgaa" "stig" "isa" + "avle" ]; }; @@ -29,6 +34,7 @@ "frankgaa" "stig" "isa" + "avle" ]; }; @@ -43,7 +49,8 @@ "simenlk" "ole" "isa" - "moritz" + "mrtz" + "avle" ]; }; @@ -55,11 +62,42 @@ "frankgaa" "stig" "isa" + "avle" ]; }; }; users.users = { + admin = pkgs.lib.mkForce { + description = "Administrator"; + home = "/home/admin"; + group = "admin"; + extraGroups = [ + "users" + "wheel" + "root" + "adm" + "cdrom" + "docker" + "fuse" + "wireshark" + "libvirtd" + "networkmanager" + "tty" + "keys" + ]; + uid = 10000; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas-3" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst" + ]; + }; + jonas = { description = "Jonas Juselius"; home = "/home/jonas"; @@ -97,6 +135,7 @@ group = "olean"; extraGroups = [ "users" + "admin" ]; uid = 1001; isNormalUser = true; @@ -134,6 +173,7 @@ "users" "wheel" "root" + "admin" ]; uid = 1003; isNormalUser = true; @@ -182,7 +222,6 @@ shell = pkgs.fish; openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik" - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzchzNR/oVJ6OgeEAmDI1jT5WTgJRk+w2XR0Euh/S+9vQ5Zlj97zRsul9KBhJLaVS2OxVl/v7gc0Cm2oyXcIacfoIF3fw4SPy/T40AHvsD6cR5H0C+q0cCFA1rroK9g4hybyc0GK+A2xs21CMfmN1C7V6jtKRz+6DKUDBcbCL+YgGo+rXxzmUPvyy9mmvx7Ae1/GZ1WoAiCWVCgE+dSmuiOtsEYVqX8eqBb73rk0wf+ijnSFSOPsVpJZciPBQ3gmyAttl1KIAoQtMxK2kFDu9i71LjnU+e4xyaJ2RuJ5747QWSne45VMShL08H56IySoY2SPboq/sOL7lg/yo9yjOCsUMjwXwzUkwvmZZX0iW5Y/yEr8WPXFI4u9rZEmAeHHt7ky+UkT3ZbXk8kcOJdALmB15iFPbtWS+7Ctrn+5R94988O874j8MNAyeZaGAeoBD75pb/EsjR6ohEV6XwBl5TQ32wexwd1sV0fTVjMFYKZGqXs8oX20o4vrbM964hG7k= adminbrede@DESKTOP-QAOIKJD" ]; }; @@ -220,10 +259,10 @@ ]; }; - moritz = { - description = "Moritz Jørg"; - home = "/home/moritz"; - group = "moritz"; + mrtz = { + description = "Moritz Jörg"; + home = "/home/mrtz"; + group = "mrtz"; extraGroups = [ "users" "wheel" @@ -243,6 +282,40 @@ ]; }; + avle = { + description = "Helge Avlesen"; + home = "/home/avle"; + group = "avle"; + extraGroups = [ + "users" + ]; + uid = 1009; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDYKIFSDy3e0HaKEhlpX0uUhWsGSU1M1ggB88/X4RXHOEf9h5J7BFCOQIyilpiD+s1zLTaMBHQTtDZpqgnKVKTk0Z5gqKpaPuf22ST5c+rxjBJzk5D09M0VHAdQr9pF8OFztiJTjXQVFfHyeNNhXWbUwTo+RpKeq3tiVDNdLcdDbdeHnCXB6VdU9f7jwPcugw6keBFK4p6RxX6ekxqG1QDFqJ7LSW5j6iF58gNTx5q2l/w4vkz7CarUtFaV+1fqsi9QEQ2ooW6hXuaxuzAB4o0htP+iPR0A0q2RlYRThYjZu2OJeoxNbGK/ny7/YZbf5GLYOYhFWytkumI//42rk+D2AFmvX4g1ksMClLoIx/9j01/9CxDt9WR+i7t83caLmGuHCnbNbnCzTa8xikpJ0Yv+jPDmksW63Zwf/YL1ZUNgD+BC3KzKwFDw2q7zO9uSsj6WS4V98IGAePyE1+xNI1NDS89OnvMsLGWEjy9MPnQzw6SbOZGvVVjMsbhwTScPvLdz3ns0n+6WM0XX4Z1WzzYL5maKtFYNiL3ntnvJAn5NZ67L/jteLiizaCKCI4dRbXg6f/toIWOQOG4KYZpkVBnURPWMF1G4eRfByD6AGLMnWCAzFfgATcqD+VmUa664VfYS7MoaS1uGagW44D87d3OiWDIOT137G//bRhUq8MOYZQ== helge.avlesen@oceanbox.io" + ]; + }; + + lilly = { + description = "Jonathan Lilly"; + home = "/home/lilly"; + group = "lilly"; + extraGroups = [ + "users" + ]; + uid = 1010; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC764LotMVJnztqHxr6hGC3ks7IXGfxBVHUMB5F0cHMsyjI4/o3yngPfFyY2k8DYxiy0shUX8aHRjH48CWBKLR4FsCnedb9OXHGUyXvjV9+/mUZtMptVx0JsLjNKn8cqCHJGGGlcfDwVqaaSixsHYNRELjgP+FSysf1C0rmPb5Hf3EFvFzCwMH13llU0eS6bj8OOGHw8fvNvkrXPKlw39xETk4IuKM1lpN+Pb+osP3L61lazrVwQJxMzl6I3I/p+M/EU4Jkx26P4eFK15JEHMP90z5ICM1kytDU9Y1LbcWJ13cAp+Mf86Z6yzdoy0j+ytMawNK2IkfgotzUuOVymdbPkWlSwcm30yyhmcjc5AQnznTdutOcvr4EOT5CmuC2vlZGkJiOTLKMd6vTLzYzC4FvDfmUC2hwvNjx2eEMUWRRL4LD57cVarckoJsqBkFF4Er6h7lRpmN1hRrVI3SWzwpAfGPDY56QWNKX2LMuMbj2zMg+fzQdLg8/qQ7tlVMD/1F9VkPocpENHPCjhhxWDBMV9tFchFb4M5tl43H2b0j6QSxnPSnFhfW+dtUWuYJ2PXwi4QcTVAvFAQIwTb6C7KuLU+wZnbQZOAzc/mqn4VyiFU+Mpbv1o+NbAKAdb3yK4RNLVyjpoJdG1iIhqmd/fOhWOGg6709FYNsSpCKYQMlzOQ== lilly@narsil.local" + ]; + }; + kraken = { description = "The Kraken"; home = "/work/kraken"; @@ -265,8 +338,24 @@ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIdcJteh9d/N1o8BbdEMRVxeMjm28saon/Oh2tV0+TYj Radovan Bast" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDjMrrzxj/BHJGWM+Wcon8RiCcMgsAKVCHl7YfopikxO isa@mare" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDYKIFSDy3e0HaKEhlpX0uUhWsGSU1M1ggB88/X4RXHOEf9h5J7BFCOQIyilpiD+s1zLTaMBHQTtDZpqgnKVKTk0Z5gqKpaPuf22ST5c+rxjBJzk5D09M0VHAdQr9pF8OFztiJTjXQVFfHyeNNhXWbUwTo+RpKeq3tiVDNdLcdDbdeHnCXB6VdU9f7jwPcugw6keBFK4p6RxX6ekxqG1QDFqJ7LSW5j6iF58gNTx5q2l/w4vkz7CarUtFaV+1fqsi9QEQ2ooW6hXuaxuzAB4o0htP+iPR0A0q2RlYRThYjZu2OJeoxNbGK/ny7/YZbf5GLYOYhFWytkumI//42rk+D2AFmvX4g1ksMClLoIx/9j01/9CxDt9WR+i7t83caLmGuHCnbNbnCzTa8xikpJ0Yv+jPDmksW63Zwf/YL1ZUNgD+BC3KzKwFDw2q7zO9uSsj6WS4V98IGAePyE1+xNI1NDS89OnvMsLGWEjy9MPnQzw6SbOZGvVVjMsbhwTScPvLdz3ns0n+6WM0XX4Z1WzzYL5maKtFYNiL3ntnvJAn5NZ67L/jteLiizaCKCI4dRbXg6f/toIWOQOG4KYZpkVBnURPWMF1G4eRfByD6AGLMnWCAzFfgATcqD+VmUa664VfYS7MoaS1uGagW44D87d3OiWDIOT137G//bRhUq8MOYZQ== helge.avlesen@oceanbox.io" ]; }; + + # kaihc = { + # description = "Kai Christensen"; + # home = "/work/kaihc"; + # group = "kraken"; + # extraGroups = []; + # uid = 3001; + # isNormalUser = true; + # createHome = true; + # useDefaultShell = true; + # openssh.authorizedKeys.keys = [ + # "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvjceOfnIOccc3Cq4gcFpfC5wK6AjjljjwKsTCep3lzRZ7olK2ylg8Y/r7LwzABvdGP700ePQP78GZOgBSj1wAkJMB+G4EAuhEiW/uVM/Q1yzo8FCNmMo8NLJxpzEEUw/SkQwpuTZ0wFoII19l78eDFeuBB80MjByd/sQbCLi+liLT++SYBudGNOk868G+IQBVhM+n5cZTavu3+dNXL/VQQah6YkKRMVYhUsHBCv0JBBsJ7YW/rpj25VlV/fKeNU5Ti90iucduyMpU8+89eRaVJYrtHnJ+mq2erWGnhMJ4YP1jXqI6ti+ydsuBLviqt8m586Y5UAuvTtGbmatVmhtsyAIn5sFApPLHikOfsEQFjvA+KBiZlPJqJFZL1S1SbUKMfY4fqKoC7CZtl8BP6CnwtfE10ZGtmiTAhDU/lXVk/wY12OXDaHJpcxF9gYVtUN0nFWwEyEoBEqU+kiuBISL1avVTsmSR49hYZC1afaWK2zvXaQUJUJVa4+TFc8azqny+N8lsjLeF7pyeWJdVBACH22wz0sU3YMW9Yv71r6kGCFXVgVaNtFpvoEBExEKFEyVJyk/94LrOPYCqflMCoi1tPITaj0zyorJdbwbSqr05veXNURrqN/WPOwSmvSrIJgkdGA9jKStfFD7RaTqBDjTK2oIUXiMXYe/6KdsUM5ne3w== kaihc@met.no" + # ]; + # }; + }; security.sudo.extraConfig = '' diff --git a/configuration.nix b/configuration.nix index 415fb13..8e24d15 100644 --- a/configuration.nix +++ b/configuration.nix @@ -30,7 +30,6 @@ let name = "frontend"; address = "10.255.241.99"; ipoib = "10.255.243.99"; - in { systemd.targets = { sleep.enable = false; @@ -66,7 +65,7 @@ in { }; features = { - desktop.enable = true; + desktop.enable = false; cachix.enable = false; host = { @@ -200,7 +199,7 @@ in { } ]; }; - interfaces.enp59s0f1np1 = { + interfaces.ens2f1np1 = { useDHCP = false; ipv4.addresses = [ { @@ -209,7 +208,7 @@ in { } ]; }; - interfaces.ibp59s0f0 = { + interfaces.ibs2f0 = { useDHCP = false; ipv4.addresses = [ { @@ -277,11 +276,23 @@ in { chmod 755 /home/jonas chmod 755 /home/stig chmod 755 /home/bast + chmod 755 /home/mrtz + chmod 755 /home/avle chmod 755 /home/simenlk chmod 755 /home/ole ''; }; + # Use nvd to get package diff before apply + system.activationScripts.system-diff = { + supportsDryActivation = true; # safe: only outputs to stdout + text = '' + export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH" + if [ -e /run/current-system ]; then + ${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true + fi + ''; + }; # ssh-rsa is deprecated, but putty/winscp users use it services.openssh.extraConfig = '' diff --git a/modules/hpc/hpc.nix b/modules/hpc/hpc.nix index 08df095..8c39e3c 100644 --- a/modules/hpc/hpc.nix +++ b/modules/hpc/hpc.nix @@ -84,6 +84,7 @@ let ]; security.sudo.extraConfig = '' %sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity + %admin ALL=(admin) NOPASSWD: ALL ''; }; @@ -115,7 +116,17 @@ let # xpmem = pkgs.callPackage ./xpmem.nix { inherit kernel; }; in { boot = { - kernelPackages = pkgs.linuxKernel.packages.linux_5_10; + #kernelPackages = pkgs.linuxKernel.packages.linux_5_10; + kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override { + argsOverride = rec { + src = pkgs.fetchurl { + url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz"; + sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid"; + }; + version = "5.10.239"; + modDirVersion = "5.10.239"; + }; + }); extraModulePackages = [ knem ]; kernelModules = [ "knem" ]; }; diff --git a/modules/k8s/default.nix b/modules/k8s/default.nix index 2443156..7ddeb05 100644 --- a/modules/k8s/default.nix +++ b/modules/k8s/default.nix @@ -223,7 +223,7 @@ let securePort = 4443; serviceClusterIpRange = "10.0.0.0/22"; extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}"; - extraSANs = cfg.master.extraSANs; + #extraSANs = cfg.master.extraSANs; verbosity = 2; etcd.servers = with builtins; @@ -469,8 +469,8 @@ in { ); imports = [ - ../overrides/kubernetes_default.nix - ../overrides/kubelet.nix + # ../overrides/kubernetes_default.nix + # ../overrides/kubelet.nix ]; } diff --git a/modules/overrides/kubelet.nix b/modules/overrides/kubelet.nix index 4d443ea..422e69e 100644 --- a/modules/overrides/kubelet.nix +++ b/modules/overrides/kubelet.nix @@ -344,10 +344,7 @@ in [ gitMinimal openssh - # TODO (#409339): remove this patch. We had to add it to avoid a mass rebuild - # for the 25.05 release. Once the staging cycle referenced in the above PR completes, - # switch back to plain util-linux. - util-linux.withPatches + util-linuxMinimal iproute2 ethtool thin-provisioning-tools diff --git a/modules/overrides/kubernetes_default.nix2 b/modules/overrides/kubernetes_default.nix2 new file mode 100644 index 0000000..86e5e07 --- /dev/null +++ b/modules/overrides/kubernetes_default.nix2 @@ -0,0 +1,358 @@ +{ + config, + lib, + options, + pkgs, + ... +}: +let + cfg = config.services.kubernetes; + opt = options.services.kubernetes; + + defaultContainerdSettings = { + version = 2; + root = "/var/lib/containerd"; + state = "/run/containerd"; + oom_score = 0; + + grpc = { + address = "/run/containerd/containerd.sock"; + }; + + plugins."io.containerd.grpc.v1.cri" = { + sandbox_image = "pause:latest"; + + cni = { + bin_dir = "/opt/cni/bin"; + max_conf_num = 0; + }; + + containerd.runtimes.runc = { + runtime_type = "io.containerd.runc.v2"; + options.SystemdCgroup = true; + }; + }; + }; + + mkKubeConfig = + name: conf: + pkgs.writeText "${name}-kubeconfig" ( + builtins.toJSON { + apiVersion = "v1"; + kind = "Config"; + clusters = [ + { + name = "local"; + cluster.certificate-authority = conf.caFile or cfg.caFile; + cluster.server = conf.server; + } + ]; + users = [ + { + inherit name; + user = { + client-certificate = conf.certFile; + client-key = conf.keyFile; + }; + } + ]; + contexts = [ + { + context = { + cluster = "local"; + user = name; + }; + name = "local"; + } + ]; + current-context = "local"; + } + ); + + caCert = secret "ca"; + + etcdEndpoints = [ "https://${cfg.masterAddress}:2379" ]; + + mkCert = + { + name, + CN, + hosts ? [ ], + fields ? { }, + action ? "", + privateKeyOwner ? "kubernetes", + privateKeyGroup ? "kubernetes", + }: + rec { + inherit + name + caCert + CN + hosts + fields + action + ; + cert = secret name; + key = secret "${name}-key"; + privateKeyOptions = { + owner = privateKeyOwner; + group = privateKeyGroup; + mode = "0600"; + path = key; + }; + }; + + secret = name: "${cfg.secretsPath}/${name}.pem"; + + mkKubeConfigOptions = prefix: { + server = lib.mkOption { + description = "${prefix} kube-apiserver server address."; + type = lib.types.str; + }; + + caFile = lib.mkOption { + description = "${prefix} certificate authority file used to connect to kube-apiserver."; + type = lib.types.nullOr lib.types.path; + default = cfg.caFile; + defaultText = lib.literalExpression "config.${opt.caFile}"; + }; + + certFile = lib.mkOption { + description = "${prefix} client certificate file used to connect to kube-apiserver."; + type = lib.types.nullOr lib.types.path; + default = null; + }; + + keyFile = lib.mkOption { + description = "${prefix} client key file used to connect to kube-apiserver."; + type = lib.types.nullOr lib.types.path; + default = null; + }; + }; +in +{ + + imports = [ + (lib.mkRemovedOptionModule [ + "services" + "kubernetes" + "addons" + "dashboard" + ] "Removed due to it being an outdated version") + (lib.mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "") + ]; + + ###### interface + + options.services.kubernetes = { + roles = lib.mkOption { + description = '' + Kubernetes role that this machine should take. + + Master role will enable etcd, apiserver, scheduler, controller manager + addon manager, flannel and proxy services. + Node role will enable flannel, docker, kubelet and proxy services. + ''; + default = [ ]; + type = lib.types.listOf ( + lib.types.enum [ + "master" + "node" + ] + ); + }; + + package = lib.mkPackageOption pkgs "kubernetes" { }; + + kubeconfig = mkKubeConfigOptions "Default kubeconfig"; + + apiserverAddress = lib.mkOption { + description = '' + Clusterwide accessible address for the kubernetes apiserver, + including protocol and optional port. + ''; + example = "https://kubernetes-apiserver.example.com:6443"; + type = lib.types.str; + }; + + caFile = lib.mkOption { + description = "Default kubernetes certificate authority"; + type = lib.types.nullOr lib.types.path; + default = null; + }; + + dataDir = lib.mkOption { + description = "Kubernetes root directory for managing kubelet files."; + default = "/var/lib/kubernetes"; + type = lib.types.path; + }; + + easyCerts = lib.mkOption { + description = "Automatically setup x509 certificates and keys for the entire cluster."; + default = false; + type = lib.types.bool; + }; + + featureGates = lib.mkOption { + description = "List set of feature gates."; + default = { }; + type = lib.types.attrsOf lib.types.bool; + }; + + masterAddress = lib.mkOption { + description = "Clusterwide available network address or hostname for the kubernetes master server."; + example = "master.example.com"; + type = lib.types.str; + }; + + path = lib.mkOption { + description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added."; + type = lib.types.listOf lib.types.package; + default = [ ]; + }; + + clusterCidr = lib.mkOption { + description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster."; + default = "10.1.0.0/16"; + type = lib.types.nullOr lib.types.str; + }; + + lib = lib.mkOption { + description = "Common functions for the kubernetes modules."; + default = { + inherit mkCert; + inherit mkKubeConfig; + inherit mkKubeConfigOptions; + }; + type = lib.types.attrs; + }; + + secretsPath = lib.mkOption { + description = "Default location for kubernetes secrets. Not a store location."; + type = lib.types.path; + default = cfg.dataDir + "/secrets"; + defaultText = lib.literalExpression '' + config.${opt.dataDir} + "/secrets" + ''; + }; + }; + + ###### implementation + + config = lib.mkMerge [ + + (lib.mkIf cfg.easyCerts { + services.kubernetes.pki.enable = lib.mkDefault true; + services.kubernetes.caFile = caCert; + }) + + (lib.mkIf (lib.elem "master" cfg.roles) { + services.kubernetes.apiserver.enable = lib.mkDefault true; + services.kubernetes.scheduler.enable = lib.mkDefault true; + services.kubernetes.controllerManager.enable = lib.mkDefault true; + services.kubernetes.addonManager.enable = lib.mkDefault true; + services.kubernetes.proxy.enable = lib.mkDefault true; + services.etcd.enable = true; # Cannot mkDefault because of flannel default options + services.kubernetes.kubelet = { + enable = lib.mkDefault true; + taints = lib.mkIf (!(lib.elem "node" cfg.roles)) { + master = { + key = "node-role.kubernetes.io/master"; + value = "true"; + effect = "NoSchedule"; + }; + }; + }; + }) + + (lib.mkIf (lib.all (el: el == "master") cfg.roles) { + # if this node is only a master make it unschedulable by default + services.kubernetes.kubelet.unschedulable = lib.mkDefault true; + }) + + (lib.mkIf (lib.elem "node" cfg.roles) { + services.kubernetes.kubelet.enable = lib.mkDefault true; + services.kubernetes.proxy.enable = lib.mkDefault true; + }) + + # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel + (lib.mkIf (cfg.roles != [ ]) { + services.kubernetes.flannel.enable = lib.mkDefault true; + services.flannel.etcd.endpoints = lib.mkDefault etcdEndpoints; + services.kubernetes.easyCerts = lib.mkDefault true; + }) + + (lib.mkIf cfg.apiserver.enable { + services.kubernetes.pki.etcClusterAdminKubeconfig = lib.mkDefault "kubernetes/cluster-admin.kubeconfig"; + services.kubernetes.apiserver.etcd.servers = lib.mkDefault etcdEndpoints; + }) + + (lib.mkIf cfg.kubelet.enable { + virtualisation.containerd = { + enable = lib.mkDefault true; + settings = lib.mapAttrsRecursive (name: lib.mkDefault) defaultContainerdSettings; + }; + }) + + (lib.mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) { + services.kubernetes.pki.certs = { + serviceAccount = mkCert { + name = "service-account"; + CN = "system:service-account-signer"; + action = '' + systemctl restart \ + kube-apiserver.service \ + kube-controller-manager.service + ''; + }; + }; + }) + + (lib.mkIf + ( + cfg.apiserver.enable + || cfg.scheduler.enable + || cfg.controllerManager.enable + || cfg.kubelet.enable + || cfg.proxy.enable + || cfg.addonManager.enable + ) + { + systemd.targets.kubernetes = { + description = "Kubernetes"; + wantedBy = [ "multi-user.target" ]; + }; + + systemd.tmpfiles.rules = [ + "d /opt/cni/bin 0755 root root -" + "d /run/kubernetes 0755 kubernetes kubernetes -" + "d ${cfg.dataDir} 0755 kubernetes kubernetes -" + ]; + + users.users.kubernetes = { + uid = config.ids.uids.kubernetes; + description = "Kubernetes user"; + group = "kubernetes"; + home = cfg.dataDir; + createHome = true; + homeMode = "755"; + }; + users.groups.kubernetes.gid = config.ids.gids.kubernetes; + + # dns addon is enabled by default + services.kubernetes.addons.dns.enable = lib.mkDefault true; + + services.kubernetes.apiserverAddress = lib.mkDefault ( + "https://${ + if cfg.apiserver.advertiseAddress != null then + cfg.apiserver.advertiseAddress + else + "${cfg.masterAddress}:${toString cfg.apiserver.securePort}" + }" + ); + } + ) + ]; + + meta.buildDocsInSandbox = false; +}