diff --git a/modules/default.nix b/modules/default.nix index 5103110..67e412d 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -21,6 +21,7 @@ with lib; ./fs ./pki/certs.nix ./gitlab-runner.nix + ./gitea-runner.nix ../nixos ]; } diff --git a/modules/gitea-runner.nix b/modules/gitea-runner.nix new file mode 100644 index 0000000..1b3b3ff --- /dev/null +++ b/modules/gitea-runner.nix @@ -0,0 +1,236 @@ +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.features.gitea-runner; + storeDeps = pkgs.runCommand "store-deps" { } '' + mkdir -p $out/bin + for dir in ${ + toString [ + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.skopeo + pkgs.openssh + pkgs.curl + pkgs.gawk + pkgs.git + pkgs.lix + pkgs.bash + pkgs.jq + pkgs.nodejs + ] + }; do + for bin in "$dir"/bin/*; do + ln -s "$bin" "$out/bin/$(basename "$bin")" + done + done + + # Add SSL CA certs + mkdir -p $out/etc/ssl/certs + cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt + ''; + + configuration = { + systemd.services = { + # everything here has no dependencies on the store + gitea-runner-nix-image = { + wantedBy = [ "multi-user.target" ]; + after = [ "podman.service" ]; + requires = [ "podman.service" ]; + path = [ + pkgs.podman + pkgs.gnutar + pkgs.shadow + pkgs.getent + ]; + # we also include etc here because the cleanup job also wants the nixuser to be present + script = '' + set -eux -o pipefail + mkdir -p etc/nix + + # Create an unpriveleged user that we can use also without the run-as-user.sh script + touch etc/passwd etc/group + groupid=$(cut -d: -f3 < <(getent group nixuser)) + userid=$(cut -d: -f3 < <(getent passwd nixuser)) + groupadd --prefix $(pwd) --gid "$groupid" nixuser + emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.' + useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser + + cat < etc/nix/nix.conf + accept-flake-config = true + experimental-features = nix-command flakes + NIX_CONFIG + + # NOTE: For private registries + # cp {config.age.secrets.nix-gitea-runner-netrc.path} etc/nix/netrc + + cat < etc/nsswitch.conf + passwd: files mymachines systemd + group: files mymachines systemd + shadow: files + + hosts: files mymachines dns myhostname + networks: files + + ethers: files + services: files + protocols: files + rpc: files + NSSWITCH + + # list the content as it will be imported into the container + tar -cv . | tar -tvf - + tar -cv . | podman import - gitea-runner-nix + ''; + serviceConfig = { + RuntimeDirectory = "gitea-runner-nix-image"; + WorkingDirectory = "/run/gitea-runner-nix-image"; + Type = "oneshot"; + RemainAfterExit = true; + }; + }; + + gitea-runner-nix = { + after = [ "gitea-runner-nix-image.service" ]; + requires = [ "gitea-runner-nix-image.service" ]; + + # Prevents gitea runner deployments + # from being restarted on a system switch, + # thus breaking a deployment. + # You'll have to restart the runner manually + # or reboot the system after a deployment! + restartIfChanged = false; + serviceConfig = { + # LoadCredential = "TOKEN_FILE:/run/gitea/gitea-runner-token"; + # EnvironmentFile = [ "$CREDENTIALS_DIRECTORY/TOKEN_FILE" ]; + EnvironmentFile = [ "/run/gitea/gitea-runner-token" ]; + + # Hardening (may overlap with DynamicUser=) + # The following options are only for optimizing output of systemd-analyze + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + # ProtectClock= adds DeviceAllow=char-rtc r + DeviceAllow = ""; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + UMask = "0066"; + ProtectProc = "invisible"; + SystemCallFilter = [ + "~@clock" + "~@cpu-emulation" + "~@module" + "~@mount" + "~@obsolete" + "~@raw-io" + "~@reboot" + "~@swap" + # needed by go? + #"~@resources" + "~@privileged" + "~capset" + "~setdomainname" + "~sethostname" + ]; + SupplementaryGroups = [ "podman" ]; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + "AF_NETLINK" + ]; + + # Needs network access + PrivateNetwork = false; + # Cannot be true due to Node + MemoryDenyWriteExecute = false; + + # The more restrictive "pid" option makes `nix` commands in CI emit + # "GC Warning: Couldn't read /proc/stat" + # You may want to set this to "pid" if not using `nix` commands + ProcSubset = "all"; + # Coverage programs for compiled code such as `cargo-tarpaulin` disable + # ASLR (address space layout randomization) which requires the + # `personality` syscall + # You may want to set this to `true` if not using coverage tooling on + # compiled code + LockPersonality = false; + + # Note that this has some interactions with the User setting; so you may + # want to consult the systemd docs if using both. + DynamicUser = true; + }; + }; + }; + + users.users.nixuser = { + group = "nixuser"; + description = "Used for running nix ci jobs"; + home = "/var/empty"; + isSystemUser = true; + }; + users.groups.nixuser = { }; + + services.gitea-actions-runner = { + instances.nix = { + enable = true; + name = "nix-runner"; + url = "https://git.svc.hel1.obx"; + # Obtaining the path to the runner token file may differ + # tokenFile should be in format TOKEN=, since it's EnvironmentFile for systemd + # tokenFile = config.age.secrets.gitea-runner-token.path; + tokenFile = ""; + labels = [ "nix:docker://gitea-runner-nix" ]; + settings = { + runner.capacity = 16; + container.options = builtins.toString [ + "-e NIX_BUILD_SHELL=/bin/bash" + "-e PAGER=cat" + "-e PATH=/bin" + "-e NIX_PATH=nixpkgs=${builtins.toString pkgs.path}" + "-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "-v /nix:/nix" + "-v ${storeDeps}/bin:/bin" + "-v ${storeDeps}/etc/ssl:/etc/ssl" + "--user nixuser" + "--device=/dev/kvm" + ]; + + # The default network that also respects our dns server settings + container.network = "host"; + container.valid_volumes = [ + "/nix" + "${storeDeps}/bin" + "${storeDeps}/etc/ssl" + ]; + }; + }; + }; + + }; +in +{ + options.features.gitea-runner = { + enable = lib.mkEnableOption "Enable Gitea runner service"; + }; + + config = lib.mkIf cfg.enable configuration; +} diff --git a/modules/gitlab-runner.nix b/modules/gitlab-runner.nix index e073d36..22fdfff 100644 --- a/modules/gitlab-runner.nix +++ b/modules/gitlab-runner.nix @@ -59,7 +59,7 @@ let cat << EOF > /etc/nix/nix.conf experimental-features = nix-command flakes pipe-operators EOF - + . ${pkgs.nix}/etc/profile.d/nix-daemon.sh ${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixos-25.05 nixpkgs ${pkgs.nix}/bin/nix-channel --update nixpkgs @@ -92,6 +92,11 @@ let }; }; }; + systemd.services.gitlab-runner = { + after = [ "podman.service" ]; + requires = [ "podman.service" ]; + serviceConfig.SupplementaryGroups = [ "podman" ]; + }; }; in { diff --git a/nixos b/nixos index d30deea..c747cb0 160000 --- a/nixos +++ b/nixos @@ -1 +1 @@ -Subproject commit d30deeae0042e309caef73c23a4120c4df2e7a6e +Subproject commit c747cb0b33d1dba18d5b69854cb6b89ed3e22cda diff --git a/tos/hashmap/default.nix b/tos/hashmap/default.nix index 33c1449..908a996 100644 --- a/tos/hashmap/default.nix +++ b/tos/hashmap/default.nix @@ -1,16 +1,20 @@ -{ pkgs, ... }: +{ + pkgs, + nixpkgs, + ... +}: { networking = { hostName = "hashmap"; domain = "obx"; search = [ "obx" ]; - firewall.allowedTCPPorts = []; + firewall.allowedTCPPorts = [ ]; firewall.extraCommands = ''''; resolvconf = { enable = false; }; nameservers = [ - "100.100.100.100" + "100.100.100.100" ]; }; @@ -104,7 +108,8 @@ networkmanager.enable = true; externalInterface = "eno2"; - docker.enable = true; + # NOTE: Use podman instead + docker.enable = false; adminAuthorizedKeys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" @@ -118,6 +123,31 @@ }; }; + virtualisation = { + podman.enable = true; + # gitlab-runner will enable this by default, but we want podman instead + docker.enable = false; + podman.autoPrune.enable = true; + # Enable Docker compatibility socket + podman.dockerCompat = true; + podman.dockerSocket.enable = true; + oci-containers.backend = "podman"; + containers.storage.settings = { + storage.graphroot = "/var/lib/containers/storage"; + storage.runroot = "/run/containers/storage"; + }; + containers.containersConf.settings = { + # podman seems to not work with systemd-resolved + containers.dns_servers = [ + "8.8.8.8" + "8.8.4.4" + ]; + }; + }; + # without this, when podman changes, it will be restarted, which will kill the build + # in the middle of restarting services and leave things in a bad state. + systemd.services.podman.restartIfChanged = false; + services.tailscale = { enable = true; useRoutingFeatures = "client"; @@ -138,8 +168,14 @@ nixpkgs.config.allowUnfreee = true; nix = { + nixPath = [ + "nixpkgs=${nixpkgs}" + ]; package = pkgs.nixVersions.stable; settings = { + nix-path = [ + "nixpkgs=${nixpkgs}" + ]; # Cleanup auto-optimise-store = true; # Keep them for debugging @@ -163,6 +199,8 @@ log-lines = 25 warn-dirty = false fallback = true + http-connections = 128 + max-substitution-jobs = 128 # Only brings pain flake-registry = "" ''; diff --git a/tos/hashmap/users.nix b/tos/hashmap/users.nix index d7e57c3..5348a24 100644 --- a/tos/hashmap/users.nix +++ b/tos/hashmap/users.nix @@ -31,6 +31,7 @@ "adm" "cdrom" "docker" + "podman" "fuse" "wireshark" "libvirtd" @@ -65,6 +66,7 @@ "root" "adm" "cdrom" + "podman" "docker" "fuse" "wireshark" @@ -95,6 +97,7 @@ "root" "adm" "cdrom" + "podman" "docker" "fuse" "wireshark" @@ -128,6 +131,7 @@ "root" "adm" "cdrom" + "podman" "docker" "fuse" "wireshark" diff --git a/tos/hive.nix b/tos/hive.nix index 63c6500..5964aad 100644 --- a/tos/hive.nix +++ b/tos/hive.nix @@ -14,9 +14,9 @@ in { ... }: { imports = [ - (import ./hashmap) - (import ../modules) (import ../nixos) + (import ../modules) + (import ./hashmap { inherit pkgs nixpkgs; }) (import "${sources.nixos-hardware}/common/cpu/intel/comet-lake") ]; @@ -44,6 +44,7 @@ in features = { gitlab-runner.enable = true; + gitea-runner.enable = true; }; services = {