log level trace

This reverts commit 60e7b841a9.

.forgejo/workflows/cd.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch: # allows manual triggering
   push:
     branches:
-      # - main
+      - main
 
 jobs:
   deploy:
@@ -13,7 +13,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up SSH key to access age secrets
+      - name: Set up SSH key for age secrets and SSH
         run: |
           env
           mkdir -p ~/.ssh
@@ -21,4 +21,4 @@ jobs:
           chmod 600 ~/.ssh/id_ed25519
 
       - name: Deploy
-        run: nix-shell --run 'nixops4 apply default'
+        run: nix-shell --run 'eval "$(ssh-agent -s)" && ssh-add ~/.ssh/id_ed25519 && SHELL=$(which bash) nixops4 apply -v default'

.forgejo/workflows/ci.yaml

@@ -10,43 +10,49 @@ on:
 
 jobs:
   check-pre-commit:
-    runs-on: native
+    runs-on: nix
     steps:
       - uses: actions/checkout@v4
       - run: nix-build -A tests
 
   check-data-model:
-    runs-on: native
+    runs-on: nix
     steps:
       - uses: actions/checkout@v4
       - run: nix-shell --run 'nix-unit ./deployment/data-model-test.nix'
 
-  check-peertube:
+  check-mastodon:
     runs-on: native
     steps:
       - uses: actions/checkout@v4
-      - run: nix-build services -A tests.peertube
+      - run: nix build .#checks.x86_64-linux.test-mastodon-service -L
+
+  check-peertube:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.test-peertube-service -L
 
   check-panel:
-    runs-on: native
+    runs-on: nix
     steps:
       - uses: actions/checkout@v4
-      - run: nix-build panel -A tests
+      - run: nix-build -A tests.panel
 
   check-deployment-basic:
-    runs-on: native
+    runs-on: nix
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.deployment-basic -L
 
   check-deployment-cli:
-    runs-on: native
+    runs-on: nix
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.deployment-cli -L
 
   check-deployment-panel:
-    runs-on: native
+    runs-on: nix
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.deployment-panel -L

.forgejo/workflows/update.yaml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   lockfile:
-    runs-on: native
+    runs-on: nix
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

default.nix

@@ -12,6 +12,7 @@ let
   inherit (pkgs) lib;
   inherit (import sources.flake-inputs) import-flake;
   inherit ((import-flake { src = ./.; }).inputs) nixops4;
+  panel = import ./panel { inherit sources system; };
   pre-commit-check =
     (import "${git-hooks}/nix" {
       inherit nixpkgs system;
@@ -71,6 +72,7 @@ in
 
   tests = {
     inherit pre-commit-check;
+    panel = panel.tests;
   };
 
   # re-export inputs so they can be overridden granularly

flake.nix

@@ -17,15 +17,20 @@
           ./infra/flake-part.nix
           ./keys/flake-part.nix
           ./secrets/flake-part.nix
+          ./services/tests/flake-part.nix
         ];
 
         perSystem =
           {
             pkgs,
             lib,
+            system,
             ...
           }:
           {
+            checks = {
+              panel = (import ./. { inherit sources system; }).tests.panel.basic;
+            };
             formatter = pkgs.nixfmt-rfc-style;
 
             pre-commit.settings.hooks =

infra/README.md

@@ -1,14 +1,13 @@
 # Infra
 
-This directory contains the definition of [the VMs](machines.md) that host our
+This directory contains the definition of [the VMs](../machines/machines.md) that host our
 infrastructure.
 
 ## Provisioning VMs with an initial configuration
 
-NOTE[Niols]: This is very manual and clunky. Two things will happen. In the near
-future, I will improve the provisioning script to make this a bit less clunky.
-In the far future, NixOps4 will be able to communicate with Proxmox directly and
-everything will become much cleaner.
+> NOTE[Niols]: This is still very manual and clunky. Two things will happen:
+> 1. In the near future, I will improve the provisioning script to make this a bit less clunky.
+> 2. In the far future, NixOps4 will be able to communicate with Proxmox directly and everything will become much cleaner.
 
 1. Choose names for your VMs. It is recommended to choose `fediXXX`, with `XXX`
    above 100. For instance, `fedi117`.
@@ -25,8 +24,7 @@ everything will become much cleaner.
    Those files need to exist during provisioning, but their content matters only
    when updating the machines' configuration.
 
-   FIXME: Remove this step by making the provisioning script not fail with the
-   public key does not exist yet.
+   > FIXME: Remove this step by making the provisioning script not fail with the public key does not exist yet.
 
 3. Run the provisioning script:
    ```
@@ -44,7 +42,7 @@ everything will become much cleaner.
    ssh fedi117.abundos.eu 'sudo cat /etc/ssh/ssh_host_ed25519_key.pub' > keys/systems/fedi117.pub
    ```
 
-   FIXME: Make the provisioning script do that for us.
+   > FIXME: Make the provisioning script do that for us.
 
 7. Regenerate the list of machines:
    ```
@@ -56,7 +54,7 @@ everything will become much cleaner.
    just enough for it to boot and be reachable. Go on to the next section to
    update the machine and put an actual configuration.
 
-   FIXME: Figure out why the full configuration isn't on the machine at this
+   > FIXME: Figure out why the full configuration isn't on the machine at this
    point and fix it.
 
 ## Updating existing VM configurations

infra/common/proxmox-qemu-vm.nix

@@ -1,15 +1,4 @@
-let
-  # pulling this in manually over from module args resolves an infinite recursion.
-  # FIXME: instead untangle `//infra/flake-part.nix` and make it stop passing wild functions.
-  # move moving towards a portable-services-like pattern where some things are submodules.
-  # Right now those wild functions are for parameterising a bunch of things,
-  # and the modular way to do that would be options --
-  # obviously you can't use those for `imports`,
-  # so one way to decouple fixpoints is to isolate them into submodules.
-  # Therefore one approach would be to try to go down the call graph,
-  # and see where what's currently a function could be a `submodule` field of something else.
-  sources = import ../../npins;
-in
+{ sources, ... }:
 {
   _class = "nixos";
 

infra/common/resource.nix

@@ -58,6 +58,8 @@ in
     users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors ++ [
       # allow our panel vm access to the test machines
       keys.panel
+      # allow continuous deployment access
+      keys.cd
     ];
 
   };

infra/flake-part.nix

@@ -27,7 +27,6 @@ let
       _module.args = {
         inherit
           inputs
-          sources
           keys
           secrets
           ;
@@ -37,6 +36,10 @@ let
         ./common/proxmox-qemu-vm.nix
       ];
 
+      nixos.specialArgs = {
+        inherit sources;
+      };
+
       imports =
         [
           ./common/resource.nix
@@ -66,17 +69,33 @@ let
     vmNames:
     { providers, ... }:
     {
-      providers.local = inputs.nixops4.modules.nixops4Provider.local;
-      resources = genAttrs vmNames (vmName: {
-        type = providers.local.exec;
-        imports = [
-          inputs.nixops4-nixos.modules.nixops4Resource.nixos
-          (makeResourceModule {
-            inherit vmName;
-            isTestVm = false;
-          })
-        ];
-      });
+      # XXX: this type merge is for adding `specialArgs` to resource modules
+      options.resources = mkOption {
+        type =
+          with lib.types;
+          lazyAttrsOf (submoduleWith {
+            class = "nixops4Resource";
+            modules = [ ];
+            # TODO(@fricklerhandwerk): we may want to pass through all of `specialArgs`
+            # once we're sure it's sane. leaving it here for better control during refactoring.
+            specialArgs = {
+              inherit sources;
+            };
+          });
+      };
+      config = {
+        providers.local = inputs.nixops4.modules.nixops4Provider.local;
+        resources = genAttrs vmNames (vmName: {
+          type = providers.local.exec;
+          imports = [
+            inputs.nixops4-nixos.modules.nixops4Resource.nixos
+            (makeResourceModule {
+              inherit vmName;
+              isTestVm = false;
+            })
+          ];
+        });
+      };
     };
   makeDeployment' = vmName: makeDeployment [ vmName ];
 

machines/dev/fedi201/fedipanel.nix

@@ -1,17 +1,17 @@
 {
   config,
+  sources,
   ...
 }:
 let
   name = "panel";
-  sources = import ../../../npins;
 in
 {
   _class = "nixos";
 
   imports = [
     (import ../../../panel { }).module
-    (import "${sources.home-manager}/nixos")
+    "${sources.home-manager}/nixos"
   ];
 
   security.acme = {

machines/dev/forgejo-ci/forgejo-actions-runner.nix

@@ -1,20 +1,79 @@
-{ pkgs, config, ... }:
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  system = builtins.currentSystem;
+  packages =
+    let
+      sources = import ../../../npins;
+      inherit (import sources.flake-inputs) import-flake;
+      inherit ((import-flake { src = ../../..; }).inputs) nixops4;
+    in
+    [
+      pkgs.coreutils
+      pkgs.findutils
+      pkgs.gnugrep
+      pkgs.gawk
+      pkgs.git
+      pkgs.nix
+      pkgs.bash
+      pkgs.jq
+      pkgs.nodejs
+      pkgs.npins
+      nixops4.packages.${system}.default
+    ];
+  storeDeps = pkgs.runCommand "store-deps" { } ''
+    mkdir -p $out/bin
+    for dir in ${toString packages}; do
+      for bin in "$dir"/bin/*; do
+        ln -s "$bin" "$out/bin/$(basename "$bin")"
+      done
+    done
+    # Add SSL CA certs
+    mkdir -p $out/etc/ssl/certs
+    cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
+  '';
+  numInstances = 5;
+in
 
 {
   _class = "nixos";
 
   services.gitea-actions-runner = {
     package = pkgs.forgejo-actions-runner;
-
-    instances.default = {
+    instances = lib.genAttrs (builtins.genList (n: "nix${builtins.toString n}") numInstances) (_: {
       enable = true;
-
       name = config.networking.fqdn;
       url = "https://git.fediversity.eu";
       tokenFile = config.age.secrets.forgejo-runner-token.path;
-
+      ## This runner supports Docker (with a default Ubuntu image) and native
+      ## modes. In native mode, it contains a few default packages.
+      labels = [
+        "nix:docker://gitea-runner-nix"
+        "docker:docker://node:16-bullseye"
+        "native:host"
+      ];
+      hostPackages = with pkgs; [
+        bash
+        git
+        nix
+        nodejs
+      ];
       settings = {
-        log.level = "info";
+        container = {
+          options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm";
+          # the default network that also respects our dns server settings
+          network = "host";
+          valid_volumes = [
+            "/nix"
+            "${storeDeps}/bin"
+            "${storeDeps}/etc/ssl"
+          ];
+        };
+        log.level = "trace";
         runner = {
           file = ".runner";
           # Take only 1 job at a time to avoid clashing NixOS tests, see #362
@@ -25,23 +84,165 @@
           fetch_interval = "2s";
         };
       };
+    });
+  };
 
-      ## This runner supports Docker (with a default Ubuntu image) and native
-      ## modes. In native mode, it contains a few default packages.
-      labels = [
-        "docker:docker://node:16-bullseye"
-        "native:host"
-      ];
-
-      hostPackages = with pkgs; [
-        bash
-        git
-        nix
-        nodejs
+  users = {
+    users.nixuser = {
+      group = "nixuser";
+      description = "Used for running nix ci jobs";
+      home = "/var/empty";
+      isSystemUser = true;
+    };
+    groups.nixuser = { };
+  };
+  virtualisation = {
+    ## For the Docker mode of the runner.
+    ## Podman seemed to get stuck on the checkout step
+    docker.enable = true;
+    containers.containersConf.settings = {
+      # podman (at least) seems to not work with systemd-resolved
+      containers.dns_servers = [
+        "8.8.8.8"
+        "8.8.4.4"
       ];
     };
   };
+  systemd.services =
+    {
+      gitea-runner-nix-image = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "docker.service" ];
+        requires = [ "docker.service" ];
+        path = [
+          pkgs.docker
+          pkgs.gnutar
+          pkgs.shadow
+          pkgs.getent
+        ];
+        # we also include etc here because the cleanup job also wants the nixuser to be present
+        script = ''
+          set -eux -o pipefail
+          mkdir -p etc/nix
 
-  ## For the Docker mode of the runner.
-  virtualisation.docker.enable = true;
+          # Create an unpriveleged user that we can use also without the run-as-user.sh script
+          touch etc/passwd etc/group
+          groupid=$(cut -d: -f3 < <(getent group nixuser))
+          userid=$(cut -d: -f3 < <(getent passwd nixuser))
+          groupadd --prefix $(pwd) --gid "$groupid" nixuser
+          emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
+          useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
+
+          cat <<NIX_CONFIG > etc/nix/nix.conf
+          accept-flake-config = true
+          experimental-features = nix-command flakes
+          NIX_CONFIG
+
+          cat <<NSSWITCH > etc/nsswitch.conf
+          passwd:    files mymachines systemd
+          group:     files mymachines systemd
+          shadow:    files
+
+          hosts:     files mymachines dns myhostname
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+          NSSWITCH
+
+          # list the content as it will be imported into the container
+          tar -cv . | tar -tvf -
+          tar -cv . | docker import - gitea-runner-nix
+        '';
+        serviceConfig = {
+          RuntimeDirectory = "gitea-runner-nix-image";
+          WorkingDirectory = "/run/gitea-runner-nix-image";
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+      };
+    }
+    // lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") numInstances) (
+      _:
+      let
+        requires = [ "gitea-runner-nix-image.service" ];
+      in
+      {
+        inherit requires;
+        after = requires;
+        # TODO: systemd confinement
+        serviceConfig = {
+          # Hardening (may overlap with DynamicUser=)
+          # The following options are only for optimizing output of systemd-analyze
+          AmbientCapabilities = "";
+          CapabilityBoundingSet = "";
+          # ProtectClock= adds DeviceAllow=char-rtc r
+          DeviceAllow = "";
+          NoNewPrivileges = true;
+          PrivateDevices = true;
+          PrivateMounts = true;
+          PrivateTmp = true;
+          PrivateUsers = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          UMask = "0066";
+          ProtectProc = "invisible";
+          SystemCallFilter = [
+            "~@clock"
+            "~@cpu-emulation"
+            "~@module"
+            "~@mount"
+            "~@obsolete"
+            "~@raw-io"
+            "~@reboot"
+            "~@swap"
+            # needed by go?
+            #"~@resources"
+            "~@privileged"
+            "~capset"
+            "~setdomainname"
+            "~sethostname"
+          ];
+          SupplementaryGroups = [ "docker" ];
+          RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+            "AF_UNIX"
+            "AF_NETLINK"
+          ];
+
+          # Needs network access
+          PrivateNetwork = false;
+          # Cannot be true due to Node
+          MemoryDenyWriteExecute = false;
+
+          # The more restrictive "pid" option makes `nix` commands in CI emit
+          # "GC Warning: Couldn't read /proc/stat"
+          # You may want to set this to "pid" if not using `nix` commands
+          ProcSubset = "all";
+          # Coverage programs for compiled code such as `cargo-tarpaulin` disable
+          # ASLR (address space layout randomization) which requires the
+          # `personality` syscall
+          # You may want to set this to `true` if not using coverage tooling on
+          # compiled code
+          LockPersonality = false;
+
+          # Note that this has some interactions with the User setting; so you may
+          # want to consult the systemd docs if using both.
+          DynamicUser = true;
+        };
+      }
+    );
 }

machines/dev/vm02116/forgejo.nix

@@ -110,4 +110,8 @@ in
       };
     };
   };
+
+  # needed to imperatively run forgejo commands e.g. to generate runner tokens.
+  # example: `sudo su - forgejo -c 'nix-shell -p forgejo --run "gitea actions generate-runner-token -C /var/lib/forgejo/custom"'`
+  users.users.forgejo.isNormalUser = true;
 }

npins/sources.json

@@ -125,6 +125,22 @@
       "url": "https://api.github.com/repos/bigskysoftware/htmx/tarball/v2.0.4",
       "hash": "1c4zm3b7ym01ijydiss4amd14mv5fbgp1n71vqjk4alc35jlnqy2"
     },
+    "nix": {
+      "type": "GitRelease",
+      "repository": {
+        "type": "GitHub",
+        "owner": "nixos",
+        "repo": "nix"
+      },
+      "pre_releases": false,
+      "version_upper_bound": null,
+      "release_prefix": null,
+      "submodules": false,
+      "version": "2.29.1",
+      "revision": "82debf3b591578eb2e7b151d2589626fad1679a2",
+      "url": "https://api.github.com/repos/nixos/nix/tarball/2.29.1",
+      "hash": "1xj5wawjw99qsyqfm3x02aydcg39rjksphnqg163plknifbzf8mc"
+    },
     "nix-unit": {
       "type": "Git",
       "repository": {

panel/default.nix

@@ -45,7 +45,7 @@ in
     '';
   };
 
-  module = import ./nix/configuration.nix;
+  module = ./nix/configuration.nix;
   tests = pkgs.callPackage ./nix/tests.nix { };
 
   # re-export inputs so they can be overridden granularly

services/default.nix

@@ -1,13 +0,0 @@
-{
-  system ? builtins.currentSystem,
-  sources ? import ../npins,
-  pkgs ? import sources.nixpkgs { inherit system; },
-  ...
-}:
-{
-  tests = {
-    mastodon = pkgs.nixosTest ./tests/mastodon.nix;
-    pixelfed-garage = pkgs.nixosTest ./tests/pixelfed-garage.nix;
-    peertube = pkgs.nixosTest ./tests/peertube.nix;
-  };
-}

services/fediversity/default.nix

@@ -49,7 +49,7 @@ in
               displayName = mkOption {
                 type = types.str;
                 description = "Name of the initial user, for humans";
-                default = config.fediversity.temp.initialUser.name;
+                default = config.fediversity.temp.initialUser.username;
               };
               email = mkOption {
                 type = types.str;

services/tests/flake-part.nix

@@ -0,0 +1,14 @@
+{ ... }:
+{
+  _class = "flake";
+
+  perSystem =
+    { pkgs, ... }:
+    {
+      checks = {
+        test-mastodon-service = pkgs.testers.runNixOSTest ./mastodon.nix;
+        test-pixelfed-garage-service = pkgs.testers.runNixOSTest ./pixelfed-garage.nix;
+        test-peertube-service = pkgs.testers.runNixOSTest ./peertube.nix;
+      };
+    };
+}

services/tests/mastodon.nix

@@ -6,7 +6,7 @@
 { pkgs, ... }:
 
 let
-  lib = pkgs.lib;
+  inherit (pkgs) lib writeText;
 
   ## FIXME: this binding was not used, but maybe we want a side-effect or something?
   # rebuildableTest = import ./rebuildableTest.nix pkgs;
@@ -69,9 +69,17 @@ in
           expect
         ];
         environment.variables = {
-          AWS_ACCESS_KEY_ID = config.fediversity.garage.ensureKeys.mastodon.id;
-          AWS_SECRET_ACCESS_KEY = config.fediversity.garage.ensureKeys.mastodon.secret;
+          AWS_ACCESS_KEY_ID = "$(cat ${config.fediversity.mastodon.s3AccessKeyFile})";
+          AWS_SECRET_ACCESS_KEY = "$(cat ${config.fediversity.mastodon.s3SecretKeyFile})";
         };
+        services.mastodon.extraEnvFiles = [
+          # generate as: cd ${pkgs.mastodon}; IGNORE_ALREADY_SET_SECRETS=true RAILS_ENV=development ${pkgs.mastodon}/bin/rails db:encryption:init
+          (writeText "rest" ''
+            ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=naGoEzeyjUmwIlmgZZmGQDWJrlWud5eX
+            ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=A0tE1VJ7S3cjaOQ58mNkhrVFY7o5NKDB
+            ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=tGHhd5Os7hLxa8QTzWwjyVLrvsj5VsCw
+          '')
+        ];
       };
   };
 

services/tests/pixelfed-garage.nix

@@ -113,6 +113,7 @@ let
 
         ${seleniumQuit}'';
 
+  dummyFile = pkgs.writeText "dummy" "dummy";
 in
 {
   name = "test-pixelfed-garage";
@@ -161,8 +162,8 @@ in
         ];
         environment.variables = {
           POST_MEDIA = ./fediversity.png;
-          AWS_ACCESS_KEY_ID = config.fediversity.garage.ensureKeys.pixelfed.id;
-          AWS_SECRET_ACCESS_KEY = config.fediversity.garage.ensureKeys.pixelfed.secret;
+          AWS_ACCESS_KEY_ID = "$(cat ${config.fediversity.pixelfed.s3AccessKeyFile})";
+          AWS_SECRET_ACCESS_KEY = "$(cat ${config.fediversity.pixelfed.s3SecretKeyFile})";
           ## without this we get frivolous errors in the logs
           MC_REGION = "garage";
         };
@@ -170,6 +171,12 @@ in
         users.users.selenium = {
           isNormalUser = true;
         };
+        fediversity.temp.initialUser = {
+          username = "dummy";
+          displayName = "dummy";
+          email = "dummy";
+          passwordFile = dummyFile;
+        };
       };
   };