From d2f9c3cb5525a4f2797fac58e06737f902159360 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Sun, 6 Jul 2025 21:25:21 +0200 Subject: [PATCH] set up ci container from clan credit: https://discourse.nixos.org/t/gitea-nix-actions-runner-setup/35279 --- .../dev/forgejo-ci/forgejo-actions-runner.nix | 288 ++++++++++++------ 1 file changed, 201 insertions(+), 87 deletions(-) diff --git a/machines/dev/forgejo-ci/forgejo-actions-runner.nix b/machines/dev/forgejo-ci/forgejo-actions-runner.nix index 00931791..352ebda0 100644 --- a/machines/dev/forgejo-ci/forgejo-actions-runner.nix +++ b/machines/dev/forgejo-ci/forgejo-actions-runner.nix @@ -1,104 +1,218 @@ +# source: https://git.clan.lol/clan/clan-infra/src/branch/main/modules/web01/gitea/actions-runner.nix { pkgs, + lib, config, - # sources, ... }: let - sources = import ../../../npins; + system = builtins.currentSystem; + packages = + let + sources = import ../../../npins; + inherit (import sources.flake-inputs) import-flake; + inherit ((import-flake { src = ../../..; }).inputs) nixops4; + in + [ + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.gawk + pkgs.git + pkgs.nix + pkgs.bash + pkgs.jq + pkgs.nodejs + pkgs.npins + nixops4.packages.${system}.default + ]; + storeDeps = pkgs.runCommand "store-deps" { } '' + mkdir -p $out/bin + for dir in ${toString packages}; do + for bin in "$dir"/bin/*; do + ln -s "$bin" "$out/bin/$(basename "$bin")" + done + done + # Add SSL CA certs + mkdir -p $out/etc/ssl/certs + cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt + ''; + numInstances = 2; in - { - _class = "nixos"; - - services.gitea-actions-runner = { - package = pkgs.forgejo-actions-runner; - - instances.default = { - enable = true; - - name = config.networking.fqdn; - url = "https://git.fediversity.eu"; - tokenFile = config.age.secrets.forgejo-runner-token.path; - - settings = { - log.level = "info"; - runner = { - file = ".runner"; - # Take only 1 job at a time to avoid clashing NixOS tests, see #362 - capacity = 1; - timeout = "3h"; - insecure = false; - fetch_timeout = "5s"; - fetch_interval = "2s"; - }; - }; - - ## This runner supports Docker (with a default Ubuntu image) and native - ## modes. In native mode, it contains a few default packages. - labels = [ - "docker:docker://node:16-bullseye" - "native:host" - ]; - - hostPackages = with pkgs; [ - bash - git - nix - nodejs + users = { + users.nixuser = { + group = "nixuser"; + description = "Used for running nix ci jobs"; + home = "/var/empty"; + isSystemUser = true; + }; + groups.nixuser = { }; + }; + virtualisation = { + podman.enable = true; + containers.containersConf.settings = { + # podman seems to not work with systemd-resolved + containers.dns_servers = [ + "8.8.8.8" + "8.8.4.4" ]; }; }; - - ## For the Docker mode of the runner. - virtualisation.docker.enable = true; - virtualisation.oci-containers.containers."buildResult" = - let - name = "nix-runner"; - tag = "latest"; - base = import (sources.nix + "/docker.nix") { - inherit pkgs; - name = "nix-ci-base"; - maxLayers = 10; - extraPkgs = with pkgs; [ - nodejs_20 # nodejs is needed for running most 3rdparty actions - # add any other pre-installed packages here - ]; - # change this is you want - channelURL = "https://nixos.org/channels/nixpkgs-23.05"; - nixConf = { - substituters = [ - "https://cache.nixos.org/" - "https://nix-community.cachix.org" - # insert any other binary caches here - ]; - trusted-public-keys = [ - "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" - # insert the public keys for those binary caches here - ]; - # allow using the new flake commands in our workflows - experimental-features = [ - "nix-command" - "flakes" + services.gitea-actions-runner.instances = + lib.genAttrs (builtins.genList (n: "nix${builtins.toString n}") numInstances) + (_: { + enable = true; + name = "nix-runner"; + url = "https://git.fediversity.eu"; + tokenFile = config.age.secrets.forgejo-runner-token.path; + labels = [ "nix:docker://gitea-runner-nix" ]; + settings = { + container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm"; + # the default network that also respects our dns server settings + container.network = "host"; + container.valid_volumes = [ + "/nix" + "${storeDeps}/bin" + "${storeDeps}/etc/ssl" ]; }; - }; - in + }); + systemd.services = { - devices = [ "/dev/kvm:/dev/kvm" ]; - image = "${name}:${tag}"; - # https://icewind.nl/entry/gitea-actions-nix/ - imageFile = pkgs.dockerTools.buildImage { - inherit name tag; - fromImage = base; - fromImageName = null; - fromImageTag = "latest"; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = [ pkgs.coreutils-full ]; - pathsToLink = [ "/bin" ]; # add coreutils (which includes sleep) to /bin + gitea-runner-nix-image = { + wantedBy = [ "multi-user.target" ]; + after = [ "podman.service" ]; + requires = [ "podman.service" ]; + path = [ + config.virtualisation.podman.package + pkgs.gnutar + pkgs.shadow + pkgs.getent + ]; + # we also include etc here because the cleanup job also wants the nixuser to be present + script = '' + set -eux -o pipefail + mkdir -p etc/nix + + # Create an unpriveleged user that we can use also without the run-as-user.sh script + touch etc/passwd etc/group + groupid=$(cut -d: -f3 < <(getent group nixuser)) + userid=$(cut -d: -f3 < <(getent passwd nixuser)) + groupadd --prefix $(pwd) --gid "$groupid" nixuser + emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.' + useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser + + cat < etc/nix/nix.conf + accept-flake-config = true + experimental-features = nix-command flakes + NIX_CONFIG + + cat < etc/nsswitch.conf + passwd: files mymachines systemd + group: files mymachines systemd + shadow: files + + hosts: files mymachines dns myhostname + networks: files + + ethers: files + services: files + protocols: files + rpc: files + NSSWITCH + + # list the content as it will be imported into the container + tar -cv . | tar -tvf - + tar -cv . | podman import - gitea-runner-nix + ''; + serviceConfig = { + RuntimeDirectory = "gitea-runner-nix-image"; + WorkingDirectory = "/run/gitea-runner-nix-image"; + Type = "oneshot"; + RemainAfterExit = true; }; }; - }; + } + // lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") numInstances) ( + _: + let + requires = [ "gitea-runner-nix-image.service" ]; + in + { + inherit requires; + after = requires; + # TODO: systemd confinement + serviceConfig = { + # Hardening (may overlap with DynamicUser=) + # The following options are only for optimizing output of systemd-analyze + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + # ProtectClock= adds DeviceAllow=char-rtc r + DeviceAllow = ""; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + UMask = "0066"; + ProtectProc = "invisible"; + SystemCallFilter = [ + "~@clock" + "~@cpu-emulation" + "~@module" + "~@mount" + "~@obsolete" + "~@raw-io" + "~@reboot" + "~@swap" + # needed by go? + #"~@resources" + "~@privileged" + "~capset" + "~setdomainname" + "~sethostname" + ]; + SupplementaryGroups = [ "podman" ]; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + "AF_NETLINK" + ]; + + # Needs network access + PrivateNetwork = false; + # Cannot be true due to Node + MemoryDenyWriteExecute = false; + + # The more restrictive "pid" option makes `nix` commands in CI emit + # "GC Warning: Couldn't read /proc/stat" + # You may want to set this to "pid" if not using `nix` commands + ProcSubset = "all"; + # Coverage programs for compiled code such as `cargo-tarpaulin` disable + # ASLR (address space layout randomization) which requires the + # `personality` syscall + # You may want to set this to `true` if not using coverage tooling on + # compiled code + LockPersonality = false; + + # Note that this has some interactions with the User setting; so you may + # want to consult the systemd docs if using both. + DynamicUser = true; + }; + } + ); }