Add login indicator

This introduces customisation to `settings.py` that
- allow controlling the relevant parameters from our systemd wrapper
  (more brittle and non-obvious than it should be, see TODOs)
- correctly configure SASS processing and static file compression
  (not as easy as it sounds)

.envrc

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
+# shellcheck shell=bash
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv --flake .)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use flake
+fi

.forgejo/workflows/ci.yaml

@@ -7,7 +7,6 @@ on:
   push:
     branches:
       - main
-      - ci/**
 
 jobs:
   check-pre-commit:
@@ -28,9 +27,3 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.peertube -L
-
-  check-pixelfed:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.pixelfed -L

.gitignore

@@ -6,7 +6,6 @@ tmp/
 .proxmox
 /.pre-commit-config.yaml
 nixos.qcow2
-.envrc
 .direnv
 result*
 .nixos-test-history

flake.nix

@@ -51,6 +51,7 @@
                 "keys"
                 "secrets"
                 "services"
+                "panel"
               ];
               files = "^((" + concatStringsSep "|" optin + ")/.*\\.nix|[^/]*\\.nix)$";
             in

panel/.envrc

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
+# shellcheck shell=bash
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use_nix
+fi

panel/.gitignore

@@ -0,0 +1,13 @@
+# Nix
+.direnv
+result*
+
+# Python
+*.pyc
+__pycache__
+
+# Django, application-specific
+db.sqlite3
+src/db.sqlite3
+src/static
+.credentials

panel/README.md

@@ -0,0 +1,46 @@
+# Fediversity Panel
+
+The Fediversity Panel is a web service for managing Fediversity deployments with a graphical user interface, written in Django.
+
+## Development
+
+- To obtain all tools related to this project, enter the development environment with `nix-shell`.
+
+  If you want to do that automatically on entering this directory:
+
+  - [Set up `direnv`](https://github.com/nix-community/nix-direnv#installation)
+  - Run `direnv allow` in the directory where repository is stored on your machine
+
+    > **Note**
+    >
+    > This is a security boundary, and allows automatically running code from this repository on your machine.
+
+- Run NixOS integration tests and Django unit tests:
+
+  ```bash
+  nix-build -A tests
+  ```
+
+- List all available Django management commands with:
+
+  ```shell-session
+  manage
+  ```
+
+- Run the server locally
+
+  ```shell-session
+  manage runserver
+  ```
+
+- Whenever you add a field in the database schema, run:
+
+  ```console
+  manage makemigrations
+  ```
+
+  Then before starting the server again, run:
+
+  ```
+  manage migrate
+  ```

panel/default.nix

@@ -0,0 +1,53 @@
+{
+  system ? builtins.currentSystem,
+  sources ? import ../npins,
+  pkgs ? import sources.nixpkgs {
+    inherit system;
+    config = { };
+    overlays = [ ];
+  },
+}:
+let
+  package =
+    let
+      callPackage = pkgs.lib.callPackageWith (pkgs // pkgs.python3.pkgs);
+    in
+    callPackage ./nix/package.nix { };
+
+  pkgs' = pkgs.extend (_final: _prev: { panel = package; });
+
+  manage = pkgs.writeScriptBin "manage" ''
+    exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
+  '';
+in
+{
+  shell = pkgs.mkShellNoCC {
+    inputsFrom = [ package ];
+    packages = [
+      pkgs.npins
+      manage
+    ];
+    env = {
+      NPINS_DIRECTORY = toString ../npins;
+    };
+    shellHook = ''
+      # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
+      # use this directory for testing with local secrets
+      mkdir -p .credentials
+      echo secret > ${builtins.toString ./.credentials}/SECRET_KEY
+      export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
+      export DATABASE_URL="sqlite:///${toString ./src}/db.sqlite3"
+    '';
+  };
+
+  tests = pkgs'.callPackage ./nix/tests.nix { };
+  inherit package;
+
+  # re-export inputs so they can be overridden granularly
+  # (they can't be accessed from the outside any other way)
+  inherit
+    sources
+    system
+    pkgs
+    ;
+}

panel/nix/configuration.nix

@@ -0,0 +1,199 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    concatStringsSep
+    mapAttrsToList
+    mkDefault
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    optionalString
+    types
+    ;
+  inherit (pkgs) writeShellApplication;
+
+  # TODO: configure the name globally for everywhere it's used
+  name = "panel";
+
+  cfg = config.services.${name};
+
+  database-url = "sqlite:////var/lib/${name}/db.sqlite3";
+
+  python-environment = pkgs.python3.withPackages (
+    ps: with ps; [
+      cfg.package
+      uvicorn
+    ]
+  );
+
+  configFile = pkgs.concatText "configuration.py" [
+    ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
+    (builtins.toFile "extra-settings.py" cfg.extra-settings)
+  ];
+
+  manage-service = writeShellApplication {
+    name = "manage";
+    text = ''exec ${cfg.package}/bin/manage.py "$@"'';
+  };
+
+  manage-admin = writeShellApplication {
+    # This allows running the `manage` command in the system environment, e.g. to initialise an admin user
+    # Executing
+    name = "manage";
+    text =
+      ''
+        systemd-run --pty \
+          --same-dir \
+          --wait \
+          --collect \
+          --service-type=exec \
+          --unit "manage-${name}.service" \
+          --property "User=${name}" \
+          --property "Group=${name}" \
+          --property "Environment=DATABASE_URL=${database-url} USER_SETTINGS_FILE=${configFile}" \
+      ''
+      + optionalString (credentials != [ ]) (
+        (concatStringsSep " \\\n" (map (cred: "--property 'LoadCredential=${cred}'") credentials)) + " \\\n"
+      )
+      + ''
+        ${lib.getExe manage-service} "$@"
+      '';
+  };
+
+  credentials = mapAttrsToList (name: secretPath: "${name}:${secretPath}") cfg.secrets;
+in
+# TODO: for a more clever and generic way of running Django services:
+#       https://git.dgnum.eu/mdebray/djangonix/
+#       unlicensed at the time of writing, but surely worth taking some inspiration from...
+{
+  options.services.${name} = {
+    enable = mkEnableOption "Service configuration for `${name}`";
+    # NOTE: this requires that the package is present in `pkgs`
+    package = mkPackageOption pkgs name { };
+    production = mkOption {
+      type = types.bool;
+      default = true;
+    };
+    restart = mkOption {
+      description = "systemd restart behavior";
+      type = types.enum [
+        "no"
+        "on-success"
+        "on-failure"
+        "on-abnormal"
+        "on-abort"
+        "always"
+      ];
+      default = "always";
+    };
+    domain = mkOption { type = types.str; };
+    host = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+    };
+    port = mkOption {
+      type = types.port;
+      default = 8000;
+    };
+    settings = mkOption {
+      type = types.attrsOf types.anything;
+      default = {
+        STATIC_ROOT = mkDefault "/var/lib/${name}/static";
+        DEBUG = mkDefault false;
+        ALLOWED_HOSTS = mkDefault [
+          cfg.domain
+          cfg.host
+          "localhost"
+          "[::1]"
+        ];
+        CSRF_TRUSTED_ORIGINS = mkDefault [ "https://${cfg.domain}" ];
+        COMPRESS_OFFLINE = true;
+        LIBSASS_OUTPUT_STYLE = "compressed";
+      };
+      description = ''
+        Django configuration as an attribute set.
+        Name-value pairs will be converted to Python variable assignments.
+      '';
+    };
+    extra-settings = mkOption {
+      type = types.lines;
+      default = "";
+      description = ''
+        Django configuration written in Python verbatim.
+        Contents will be appended to the definitions in `settings`.
+      '';
+    };
+    secrets = mkOption {
+      type = types.attrsOf types.path;
+      default = { };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ manage-admin ];
+
+    services = {
+      nginx.enable = true;
+      nginx.virtualHosts = {
+        ${cfg.domain} =
+          {
+            locations = {
+              "/".proxyPass = "http://localhost:${toString cfg.port}";
+              "/static/".alias = "/var/lib/${name}/static/";
+            };
+          }
+          // lib.optionalAttrs cfg.production {
+            enableACME = true;
+            forceSSL = true;
+          };
+      };
+    };
+
+    users.users.${name} = {
+      isSystemUser = true;
+      group = name;
+    };
+
+    users.groups.${name} = { };
+    systemd.services.${name} = {
+      description = "${name} ASGI server";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [
+        python-environment
+        manage-service
+      ];
+      preStart = ''
+        # Auto-migrate on first run or if the package has changed
+        versionFile="/var/lib/${name}/package-version"
+        if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
+          manage migrate --no-input
+          manage collectstatic --no-input --clear
+          manage compress --force
+          echo ${cfg.package} > "$versionFile"
+        fi
+      '';
+      script = ''
+        uvicorn ${name}.asgi:application --host ${cfg.host} --port ${toString cfg.port}
+      '';
+      serviceConfig = {
+        Restart = "always";
+        User = name;
+        WorkingDirectory = "/var/lib/${name}";
+        StateDirectory = name;
+        RuntimeDirectory = name;
+        LogsDirectory = name;
+      } // lib.optionalAttrs (credentials != [ ]) { LoadCredential = credentials; };
+      environment = {
+        USER_SETTINGS_FILE = "${configFile}";
+        DATABASE_URL = database-url;
+      };
+    };
+  };
+}

panel/nix/package.nix

@@ -0,0 +1,57 @@
+{
+  lib,
+  buildPythonPackage,
+  setuptools,
+  django_4,
+  django-compressor,
+  django-libsass,
+  dj-database-url,
+}:
+let
+  src =
+    with lib.fileset;
+    toSource {
+      root = ../src;
+      fileset = intersection (gitTracked ../../.) ../src;
+    };
+  pyproject = with lib; fromTOML pyproject-toml;
+  # TODO: define this globally
+  name = "panel";
+  # TODO: we may want this in a file so it's easier to read statically
+  version = "0.0.0";
+  pyproject-toml = ''
+    [project]
+    name = "Fediversity-Panel"
+    version = "${version}"
+
+    [tool.setuptools]
+    packages = [ "${name}" ]
+    include-package-data = true
+  '';
+in
+buildPythonPackage {
+  pname = name;
+  inherit (pyproject.project) version;
+  pyproject = true;
+  inherit src;
+
+  preBuild = ''
+    echo "recursive-include ${name} *" > MANIFEST.in
+    cp ${builtins.toFile "source" pyproject-toml} pyproject.toml
+  '';
+
+  propagatedBuildInputs = [
+    setuptools
+    django_4
+    django-compressor
+    django-libsass
+    dj-database-url
+  ];
+
+  postInstall = ''
+    mkdir -p $out/bin
+    cp -v ${src}/manage.py $out/bin/manage.py
+    chmod +x $out/bin/manage.py
+    wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
+  '';
+}

panel/nix/tests.nix

@@ -0,0 +1,62 @@
+{ lib, pkgs }:
+let
+  # TODO: specify project/service name globally
+  name = "panel";
+  defaults = {
+    services.${name} = {
+      enable = true;
+      production = false;
+      restart = "no";
+      domain = "example.com";
+      secrets = {
+        SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret";
+      };
+    };
+
+    virtualisation = {
+      memorySize = 2048;
+      cores = 2;
+    };
+  };
+in
+lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) {
+  application-tests = {
+    inherit defaults;
+    nodes.server = _: { imports = [ ./configuration.nix ]; };
+    # run all application-level tests managed by Django
+    # https://docs.djangoproject.com/en/5.0/topics/testing/overview/
+    testScript = ''
+      server.succeed("manage test")
+    '';
+  };
+  admin = {
+    inherit defaults;
+    nodes.server = _: { imports = [ ./configuration.nix ]; };
+    # check that the admin interface is served
+    testScript = ''
+      server.wait_for_unit("multi-user.target")
+      server.wait_for_unit("${name}.service")
+      server.wait_for_open_port(8000)
+      server.succeed("curl --fail -L -H 'Host: example.org' http://localhost/admin")
+    '';
+  };
+
+  sass-processing = {
+    inherit defaults;
+    nodes.server = _: { imports = [ ./configuration.nix ]; };
+    extraPythonPackages = ps: with ps; [ beautifulsoup4 ];
+    skipTypeCheck = true;
+    # check that stylesheets are pre-processed and served
+    testScript = ''
+      from bs4 import BeautifulSoup
+      server.wait_for_unit("multi-user.target")
+      server.wait_for_unit("${name}.service")
+      server.wait_for_open_port(8000)
+      stdout = server.succeed("curl --fail -H 'Host: example.org' http://localhost")
+      # the CSS is auto-generated with a hash in the file name
+      html = BeautifulSoup(stdout, 'html.parser')
+      css = html.find('link', type="text/css")['href']
+      server.succeed(f"curl --fail -H 'Host: example.org' http://localhost/{css}")
+    '';
+  };
+}

panel/shell.nix

@@ -0,0 +1 @@
+(import ./. { }).shell

panel/src/manage.py

@@ -0,0 +1,22 @@
+#!/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/bin/python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

panel/src/panel/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for panel project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
+
+application = get_asgi_application()

panel/src/panel/settings.py

@@ -0,0 +1,171 @@
+"""
+Django settings for panel project.
+
+Generated by 'django-admin startproject' using Django 4.2.16.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.2/ref/settings/
+"""
+
+import sys
+import os
+import importlib.util
+import dj_database_url
+
+from os import environ as env
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+
+def get_secret(name: str, encoding: str = "utf-8") -> str:
+    credentials_dir = env.get("CREDENTIALS_DIRECTORY")
+
+    if credentials_dir is None:
+        raise RuntimeError("No credentials directory available.")
+
+    try:
+        with open(f"{credentials_dir}/{name}", encoding=encoding) as f:
+            secret = f.read().removesuffix("\n")
+    except FileNotFoundError:
+        raise RuntimeError(f"No secret named {name} found in {credentials_dir}.")
+
+    return secret
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = get_secret("SECRET_KEY")
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "panel",
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'compressor',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'panel.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'panel.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
+# https://github.com/jazzband/dj-database-url
+
+DATABASES = {
+    'default': dj_database_url.config(),
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/4.2/howto/static-files/
+
+STATIC_URL = 'static/'
+
+STATIC_ROOT = os.path.join(BASE_DIR, "static/")
+
+STATICFILES_FINDERS = [
+    "django.contrib.staticfiles.finders.FileSystemFinder",
+    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
+    "compressor.finders.CompressorFinder",
+]
+
+STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
+
+COMPRESS_PRECOMPILERS = [
+    ("text/x-sass", "django_libsass.SassCompiler"),
+]
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+# Customization via user settings
+# This must be at the end, as it must be able to override the above
+# TODO: we may want to do this with a flat environment instead, and get all values from `os.environ.get()`.
+#       this would make it more obvious which moving parts there are, if that environment is specified for development/staging/production in a visible place.
+user_settings_file = env.get("USER_SETTINGS_FILE", None)
+if user_settings_file is not None:
+    spec = importlib.util.spec_from_file_location("user_settings", user_settings_file)
+    if spec is None or spec.loader is None:
+        raise RuntimeError("User settings specification failed!")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules["user_settings"] = module
+    from user_settings import *  # noqa: F403 # pyright: ignore [reportMissingImports]

panel/src/panel/static/style.sass

@@ -0,0 +1,5 @@
+body
+  padding: 0
+  margin: 0
+  font-family: sans-serif
+  box-sizing: border-box

panel/src/panel/templates/base.html

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>{% block title %}Fediversity Panel{% endblock %}</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+
+    {% load compress %}
+    {% compress css %}
+    <link rel="stylesheet" type="text/x-sass" href="/static/style.sass" />
+    {% endcompress %}
+
+    {% block extra_head %}{% endblock extra_head %}
+  </head>
+
+  <body>
+    <header>
+    {% if user.is_authenticated %}
+      <p>Welcome, {{ user.username }}!</p>
+    {% else %}
+      <p>You are not logged in.</p>
+    {% endif %}
+    </header>
+    {% block navigation %}
+    <nav>
+    </nav>
+    {% endblock navigation %}
+
+    {% block layout %}
+    <article>
+      {% block content %}{% endblock content %}
+    </article>
+    {% endblock layout %}
+  </body>
+</html>

panel/src/panel/templates/index.html

@@ -0,0 +1,7 @@
+{% extends "base.html" %}
+
+{% block content %}
+<h1>Fediversity Panel</h1>
+
+<p>Hello world!</p>
+{% endblock %}

panel/src/panel/urls.py

@@ -0,0 +1,24 @@
+"""
+URL configuration for panel project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/4.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path
+from panel import views
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+    path('', views.Index.as_view(), name='index'),
+]

panel/src/panel/views.py

@@ -0,0 +1,4 @@
+from django.views.generic import TemplateView
+
+class Index(TemplateView):
+    template_name = 'index.html'

panel/src/panel/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for panel project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
+
+application = get_wsgi_application()

services/fediversity/pixelfed.nix

@@ -13,14 +13,6 @@ in
 }:
 
 lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
-
-  ## Pixelfed as packaged in nixpkgs has a permission issue that prevents Nginx
-  ## from being able to serving the images. We fix it here, but this should be
-  ## upstreamed. See https://github.com/NixOS/nixpkgs/issues/235147
-  services.pixelfed.package = pkgs.pixelfed.overrideAttrs (old: {
-    patches = (old.patches or [ ]) ++ [ ./pixelfed-group-permissions.patch ];
-  });
-
   services.garage = {
     ensureBuckets = {
       pixelfed = {
@@ -69,8 +61,6 @@ lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
     };
   };
 
-  users.users.nginx.extraGroups = [ "pixelfed" ];
-
   services.pixelfed.settings = {
     ## NOTE: This depends on the targets, eg. universities might want control
     ## over who has an account. We probably want a universal

services/flake-part.nix

@@ -8,8 +8,8 @@
     {
       checks = {
         mastodon = import ./tests/mastodon.nix { inherit self pkgs; };
+        pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit self pkgs; };
         peertube = import ./tests/peertube.nix { inherit self pkgs; };
-        pixelfed = import ./tests/pixelfed.nix { inherit self pkgs; };
       };
     };
 }

services/tests/mastodon.nix

@@ -8,6 +8,9 @@
 let
   lib = pkgs.lib;
 
+  ## FIXME: this binding was not used, but maybe we want a side-effect or something?
+  # rebuildableTest = import ./rebuildableTest.nix pkgs;
+
   testImage = pkgs.copyPathToStore ./green.png;
   testImageColour = "#00FF00";
 

services/tests/peertube.nix

@@ -1,6 +1,4 @@
 ## This file is a basic test of Peertube functionalities.
-##
-## NOTE: This test needs Peertube >= 6.3.
 
 { pkgs, self }:
 
@@ -14,11 +12,7 @@ let
     pkgs.writers.writePython3Bin "post-video-in-browser"
       {
         libraries = with pkgs.python3Packages; [ selenium ];
-        flakeIgnore = [
+        flakeIgnore = [ "E501" ]; # welcome to the 21st century
-          "E302"
-          "E305"
-          "E501" # welcome to the 21st century
-        ];
       }
       ''
         import sys
@@ -30,18 +24,31 @@ let
         from selenium.webdriver.support.ui import WebDriverWait
         from selenium.common.exceptions import NoSuchElementException
 
-        print("Create and configure driver...", file=sys.stderr)
         options = Options()
+        print("########################################", file=sys.stderr)
+        print("A", file=sys.stderr)
         options.add_argument("--headless")
+        print("########################################", file=sys.stderr)
+        print("B", file=sys.stderr)
         service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")
+        print("########################################", file=sys.stderr)
+        print("C", file=sys.stderr)
         driver = webdriver.Firefox(options=options, service=service)
+        print("########################################", file=sys.stderr)
+        print("D", file=sys.stderr)
         driver.set_window_size(4096, 2160)
+        print("########################################", file=sys.stderr)
+        print("E", file=sys.stderr)
         driver.implicitly_wait(360)
+        print("########################################", file=sys.stderr)
+        print("F", file=sys.stderr)
         wait = WebDriverWait(driver, timeout=360, poll_frequency=10)
+        print("########################################", file=sys.stderr)
 
         ############################################################
         # Login
 
+
         def load(driver, page):
             print(f"Loading page {page}...", file=sys.stderr)
             driver.get(page)
@@ -77,6 +84,7 @@ let
 
             print(f"Done loading page {page}.", file=sys.stderr)
 
+
         ############################################################
         # Upload video and take a screenshot
 
@@ -107,9 +115,9 @@ let
         # driver.find_element(By.XPATH, "//*[contains(text(), 'Failed to play video')]")
         #
         # def detect_image_in_screen(d):
-        #     print("  Taking a screenshot...", file=sys.stderr)
+        #     print("Taking a screenshot...", file=sys.stderr)
         #     d.save_screenshot("/screenshot.png")
-        #     print("  Checking it...", file=sys.stderr)
+        #     print("Checking it...", file=sys.stderr)
         #     displayed_colours = subprocess.run(
         #         [
         #             "magick",
@@ -124,10 +132,7 @@ let
         #         text=True,
         #         check=True,
         #     ).stdout
-        #     result = bool(re.search("${testVideoColour}", displayed_colours, re.IGNORECASE))
+        #     return bool(re.match(".*#${testVideoColour}.*", displayed_colours, re.S))
-        #     if not result:
-        #         print("  Could not find the video in the screenshot.", file=sys.stderr)
-        #     return result
         #
         # print("Wait until the image shows in screen...", file=sys.stderr)
         # wait.until(detect_image_in_screen)
@@ -170,6 +175,11 @@ pkgs.nixosTest {
           ../vm/interactive-vm.nix
         ];
 
+        virtualisation = {
+          memorySize = lib.mkVMOverride 8192;
+          cores = 8;
+        };
+
         environment.systemPackages = with pkgs; [
           python3
           firefox-unwrapped
@@ -182,15 +192,15 @@ pkgs.nixosTest {
           expect
         ];
 
+        ## FIXME: The CI is very slow, so the default timeout of 120s is not
+        ## good enough. We bump it drastically.
+        systemd.services.postgresql.serviceConfig.TimeoutSec = lib.mkForce 3600;
+
         environment.variables = {
           AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.peertube.id;
           AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.peertube.secret;
           PT_INITIAL_ROOT_PASSWORD = "testtest";
         };
-
-        ## FIXME: The CI is very slow, so the default timeout of 120s is not
-        ## good enough. We bump it drastically.
-        systemd.services.postgresql.serviceConfig.TimeoutSec = lib.mkForce 3600;
       };
   };
 
@@ -201,6 +211,7 @@ pkgs.nixosTest {
 
       # FIXME: I think this trick to look for a password can be replaced by
       # services.peertube.serviceEnvironmentFile.PT_INITIAL_ROOT_PASSWORD=testtest
+
       with subtest("Peertube starts"):
         server.wait_for_unit("peertube.service")
         root_password = server.succeed("acquire-root-password").rstrip()

services/tests/pixelfed-garage.nix

@@ -0,0 +1,222 @@
+{ pkgs, self }:
+let
+  lib = pkgs.lib;
+
+  ## FIXME: this binding was not used but maybe we want a side effect or something?
+  # rebuildableTest = import ./rebuildableTest.nix pkgs;
+
+  email = "test@test.com";
+  password = "testtest";
+
+  # FIXME: Replace all the By.XPATH by By.CSS_SELECTOR.
+
+  seleniumImports = ''
+    import sys
+    from selenium import webdriver
+    from selenium.webdriver.common.by import By
+    from selenium.webdriver.chrome.options import Options
+  '';
+
+  seleniumSetup = ''
+    print("Create and configure driver...", file=sys.stderr)
+    options = Options()
+    # options.add_argument("--headless=new")
+    service = webdriver.ChromeService(executable_path="${lib.getExe pkgs.chromedriver}")  # noqa: E501
+    driver = webdriver.Chrome(options=options, service=service)
+    driver.implicitly_wait(30)
+    driver.set_window_size(1280, 960)
+  '';
+
+  seleniumPixelfedLogin = ''
+    print("Open login page...", file=sys.stderr)
+    driver.get("http://pixelfed.localhost/login")
+    print("Enter email...", file=sys.stderr)
+    driver.find_element(By.ID, "email").send_keys("${email}")
+    print("Enter password...", file=sys.stderr)
+    driver.find_element(By.ID, "password").send_keys("${password}")
+    # FIXME: This is disgusting. Find instead the input type submit in the form
+    # with action ending in "/login".
+    print("Click “Login” button...", file=sys.stderr)
+    driver.find_element(By.XPATH, "//button[normalize-space()='Login']").click()
+  '';
+
+  ## NOTE: `path` must be a valid python string, either a variable or _quoted_.
+  seleniumTakeScreenshot = path: ''
+    print("Take screenshot...", file=sys.stderr)
+    if not driver.save_screenshot(${path}):
+        raise Exception("selenium could not save screenshot")
+  '';
+
+  seleniumQuit = ''
+    print("Quitting...", file=sys.stderr)
+    driver.quit()
+  '';
+
+  seleniumScriptPostPicture =
+    pkgs.writers.writePython3Bin "selenium-script-post-picture"
+      { libraries = with pkgs.python3Packages; [ selenium ]; }
+      ''
+        import os
+        import time
+        ${seleniumImports}
+        from selenium.webdriver.support.wait import WebDriverWait
+
+        ${seleniumSetup}
+        ${seleniumPixelfedLogin}
+        time.sleep(3)
+
+        media_path = os.environ['POST_MEDIA']
+
+        # Find the new post form, fill it in with our pictureand a caption.
+        print("Click on “Create New Post”...", file=sys.stderr)
+        driver.find_element(By.LINK_TEXT, "Create New Post").click()
+        print("Add file to input element...", file=sys.stderr)
+        driver.find_element(By.XPATH, "//input[@type='file']").send_keys(media_path)
+        print("Add a caption", file=sys.stderr)
+        driver.find_element(By.CSS_SELECTOR, ".media-body textarea").send_keys(
+            "Fediversity test of image upload to pixelfed with garage storage."
+        )
+        time.sleep(3)
+        print("Click on “Post” button...", file=sys.stderr)
+        driver.find_element(By.LINK_TEXT, "Post").click()
+
+        # Wait until the post loads, and in particular its picture, then take a
+        # screenshot of the whole page.
+        print("Wait for post and image to be loaded...", file=sys.stderr)
+        img = driver.find_element(
+            By.XPATH,
+            "//div[@class='timeline-status-component-content']//img"
+        )
+        WebDriverWait(driver, timeout=10).until(
+            lambda d: d.execute_script("return arguments[0].complete", img)
+        )
+        time.sleep(3)
+
+        ${seleniumTakeScreenshot "\"/home/selenium/screenshot.png\""}
+        ${seleniumQuit}'';
+
+  seleniumScriptGetSrc =
+    pkgs.writers.writePython3Bin "selenium-script-get-src"
+      { libraries = with pkgs.python3Packages; [ selenium ]; }
+      ''
+        ${seleniumImports}
+        ${seleniumSetup}
+        ${seleniumPixelfedLogin}
+
+        img = driver.find_element(
+            By.XPATH,
+            "//div[@class='timeline-status-component-content']//img"
+        )
+        # REVIEW: Need to wait for it to be loaded?
+        print(img.get_attribute('src'))
+
+        ${seleniumQuit}'';
+
+in
+pkgs.nixosTest {
+  name = "test-pixelfed-garage";
+
+  nodes = {
+    server =
+      { config, ... }:
+      {
+
+        services = {
+          xserver = {
+            enable = true;
+            displayManager.lightdm.enable = true;
+            desktopManager.lxqt.enable = true;
+          };
+
+          displayManager.autoLogin = {
+            enable = true;
+            user = "selenium";
+          };
+        };
+        virtualisation.resolution = {
+          x = 1680;
+          y = 1050;
+        };
+
+        virtualisation = {
+          memorySize = lib.mkVMOverride 8192;
+          cores = 8;
+        };
+        imports = with self.nixosModules; [
+          fediversity
+          ../vm/garage-vm.nix
+          ../vm/pixelfed-vm.nix
+        ];
+        # TODO: pair down
+        environment.systemPackages = with pkgs; [
+          python3
+          chromium
+          chromedriver
+          xh
+          seleniumScriptPostPicture
+          seleniumScriptGetSrc
+          helix
+          imagemagick
+        ];
+        environment.variables = {
+          POST_MEDIA = ./fediversity.png;
+          AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
+          AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
+          ## without this we get frivolous errors in the logs
+          MC_REGION = "garage";
+        };
+        # chrome does not like being run as root
+        users.users.selenium = {
+          isNormalUser = true;
+        };
+      };
+  };
+
+  testScript =
+    { nodes, ... }:
+    ''
+      import re
+
+      server.start()
+
+      with subtest("Pixelfed starts"):
+        server.wait_for_unit("phpfpm-pixelfed.service")
+
+      with subtest("Account creation"):
+        server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
+
+      # NOTE: This could in theory give a false positive if pixelfed changes it's
+      # colorscheme to include pure green. (see same problem in mastodon-garage.nix).
+      # TODO: For instance: post a red image and check that the green pixel IS NOT
+      # there, then post a green image and check that the green pixel IS there.
+
+      with subtest("Image displays"):
+        server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
+        server.copy_from_vm("/home/selenium/screenshot.png", "")
+        displayed_colors = server.succeed("magick /home/selenium/screenshot.png -define histogram:unique-colors=true -format %c histogram:info:")
+        # check that the green image displayed somewhere
+        image_check = re.match(".*#FF0500.*", displayed_colors, re.S)
+        if image_check is None:
+          raise Exception("cannot detect the uploaded image on pixelfed page.")
+
+      with subtest("access garage"):
+        server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
+        server.succeed("mc ls garage/pixelfed")
+
+      with subtest("access image in garage"):
+        image = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
+        image = image.rstrip()
+        if image == "":
+          raise Exception("image posted to Pixelfed did not get stored in garage")
+        server.succeed(f"mc cat {image} >/garage-image.png")
+        garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.png")
+        image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA")
+        if garage_image_hash != image_hash:
+          raise Exception("image stored in garage did not match image uploaded")
+
+      with subtest("Check that image comes from garage"):
+        src = server.succeed("su - selenium -c 'selenium-script-get-src ${email} ${password}'")
+        if not src.startswith("${nodes.server.fediversity.internal.garage.web.urlForBucket "pixelfed"}"):
+          raise Exception("image does not come from garage")
+    '';
+}

services/tests/pixelfed.nix

@@ -1,191 +0,0 @@
-{ pkgs, self }:
-
-let
-  lib = pkgs.lib;
-
-  email = "test@test.com";
-  password = "testtest";
-
-  testPicture = pkgs.copyPathToStore ./green.png;
-  testPictureColour = "#00FF00";
-
-  seleniumScriptPostPicture =
-    garageBucketUrl:
-    pkgs.writers.writePython3Bin "selenium-script-post-picture"
-      {
-        libraries = with pkgs.python3Packages; [ selenium ];
-        flakeIgnore = [
-          "E302"
-          "E305"
-          "E501" # welcome to the 21st century
-        ];
-      }
-      ''
-        import re
-        import subprocess
-        import sys
-        from selenium import webdriver
-        from selenium.webdriver.common.by import By
-        from selenium.webdriver.firefox.options import Options
-        from selenium.webdriver.support.ui import WebDriverWait
-
-        print("Create and configure driver...", file=sys.stderr)
-        options = Options()
-        options.add_argument("--headless")
-        service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")
-        driver = webdriver.Firefox(options=options, service=service)
-        driver.set_window_size(4096, 2160)
-        driver.implicitly_wait(360)
-        wait = WebDriverWait(driver, timeout=1080, poll_frequency=120)
-
-        ############################################################
-        # Login
-
-        print("Open login page...", file=sys.stderr)
-        driver.get("http://pixelfed.localhost/login")
-        print("Enter email...", file=sys.stderr)
-        driver.find_element(By.ID, "email").send_keys("${email}")
-        print("Enter password...", file=sys.stderr)
-        driver.find_element(By.ID, "password").send_keys("${password}")
-        print("Click “Login” button...", file=sys.stderr)
-        driver.find_element(By.XPATH, "//button[normalize-space()='Login']").click()
-
-        ############################################################
-        # Post picture
-
-        # Find the new post form, fill it in with our picture.
-        print("Click on “Create New Post”...", file=sys.stderr)
-        driver.find_element(By.LINK_TEXT, "Create New Post").click()
-        print("Add file to input element...", file=sys.stderr)
-        driver.find_element(By.XPATH, "//input[@type='file']").send_keys("${testPicture}")
-        print("Click on “Post” button...", file=sys.stderr)
-        driver.find_element(By.LINK_TEXT, "Post").click()
-
-        # Wait until the post loads, and in particular its picture, then take a
-        # screenshot of the whole page.
-        print("Wait for post and picture to be loaded...", file=sys.stderr)
-        img = driver.find_element(By.XPATH, "//div[@class='timeline-status-component-content']//img")
-        wait.until(lambda d: d.execute_script("return arguments[0].complete", img))
-
-        ############################################################
-        # Check that the picture actually shows
-
-        def detect_picture_in_screen(d):
-            print("  Taking a screenshot...", file=sys.stderr)
-            d.save_screenshot("/home/selenium/screenshot.png")
-            print("  Checking it...", file=sys.stderr)
-            displayed_colours = subprocess.run(
-                [
-                    "magick",
-                    "/home/selenium/screenshot.png",
-                    "-define",
-                    "histogram:unique-colors=true",
-                    "-format",
-                    "%c",
-                    "histogram:info:",
-                ],
-                capture_output=True,
-                text=True,
-                check=True,
-            ).stdout
-            result = bool(re.search("${testPictureColour}", displayed_colours, re.IGNORECASE))
-            if not result:
-                print("  Could not find the picture in the screenshot.", file=sys.stderr)
-            return result
-
-        print("Wait until the picture shows in screen...", file=sys.stderr)
-        wait.until(detect_picture_in_screen)
-        print("Picture detected!", file=sys.stderr)
-
-        ############################################################
-        # Check that the picture gets to Garage
-
-        def detect_src_in_garage(d):
-            print("  Reload the timeline...", file=sys.stderr)
-            driver.get("http://pixelfed.localhost/")
-            print("  Getting the picture's src and checking it...", file=sys.stderr)
-            img = driver.find_element(By.XPATH, "//div[@class='timeline-status-component-content']//img")
-            result = img.get_attribute('src').startswith("${garageBucketUrl}")
-            if not result:
-                print("  The picture's src does not point to Garage.", file=sys.stderr)
-            return result
-
-        print("Wait until the picture's src points to Garage...", file=sys.stderr)
-        wait.until(detect_src_in_garage)
-        print("Picture's src points to Garage!", file=sys.stderr)
-
-        ############################################################
-
-        print("Done; bye!", file=sys.stderr)
-        driver.close()
-      '';
-
-in
-
-pkgs.nixosTest {
-  name = "pixelfed";
-
-  nodes = {
-    server =
-      { config, ... }:
-      {
-        imports = with self.nixosModules; [
-          fediversity
-          ../vm/garage-vm.nix
-          ../vm/pixelfed-vm.nix
-          ../vm/interactive-vm.nix
-        ];
-
-        environment.systemPackages = with pkgs; [
-          python3
-          firefox-unwrapped
-          geckodriver
-          xh
-          (seleniumScriptPostPicture (config.fediversity.internal.garage.web.urlForBucket "pixelfed"))
-          helix
-          imagemagick
-        ];
-
-        environment.variables = {
-          AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
-          AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
-          ## without this we get frivolous errors in the logs
-          MC_REGION = "garage";
-        };
-
-        # Do not run Selenium scripts as root
-        users.users.selenium.isNormalUser = true;
-      };
-  };
-
-  testScript =
-    { nodes, ... }:
-    ''
-      server.start()
-
-      with subtest("Pixelfed starts"):
-        server.wait_for_unit("phpfpm-pixelfed.service")
-        server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
-
-      # NOTE: This could in theory give a false positive if pixelfed changes it's
-      # colorscheme to include pure green. (see same problem in mastodon-garage.nix).
-      # TODO: For instance: post a red picture and check that the green pixel IS NOT
-      # there, then post a green picture and check that the green pixel IS there.
-
-      with subtest("Post an picture in the browser"):
-        server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
-        server.copy_from_vm("/home/selenium/screenshot.png", "")
-
-      with subtest("Find picture in garage"):
-        server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
-        picture = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
-        picture = picture.rstrip()
-        if picture == "":
-          raise Exception("Could not find any _thumb.png picture stored in Garage")
-        server.succeed(f"mc cat {picture} > /picture.png")
-        garage_hash = server.succeed("identify -quiet -format '%#' /picture.png")
-        hash = server.succeed("identify -quiet -format '%#' ${testPicture}")
-        if garage_hash != hash:
-          raise Exception("The picture stored in Garage does not correspond to the original one.")
-    '';
-}

services/vm/garage-vm.nix

@@ -28,6 +28,7 @@ in
       inherit value;
     }) (filterAttrs (_: { website, ... }: website) cfg.ensureBuckets);
 
+  virtualisation.diskSize = 2048;
   virtualisation.forwardPorts = [
     {
       from = "host";

services/vm/interactive-vm.nix

@@ -1,19 +1,9 @@
-{ pkgs, lib, ... }:
+# customize nixos-rebuild build-vm to be a bit more convenient
-
+{ pkgs, ... }:
-let
-  inherit (lib) mkForce;
-
-in
-
 {
-  users = {
+  # let us log in
-    mutableUsers = false;
+  users.mutableUsers = false;
-    users.root = {
+  users.users.root.hashedPassword = "";
-      hashedPassword = "";
-      hashedPasswordFile = mkForce null;
-    };
-  };
-
   services.openssh = {
     enable = true;
     settings = {
@@ -23,11 +13,16 @@ in
     };
   };
 
+  # automatically log in
   services.getty.autologinUser = "root";
-
+  services.getty.helpLine = ''
+    Type `C-a c` to access the qemu console
+    Type `C-a x` to quit
+  '';
+  # access to convenient things
   environment.systemPackages = with pkgs; [
-    python3
     w3m
+    python3
     xterm # for `resize`
   ];
   environment.loginShellInit = ''
@@ -37,12 +32,9 @@ in
     extra-experimental-features = nix-command flakes
   '';
 
-  virtualisation = {
+  virtualisation.memorySize = 2048;
-    memorySize = 8192;
-    diskSize = 2048;
-    cores = 8;
 
-    forwardPorts = [
+  virtualisation.forwardPorts = [
     {
       from = "host";
       host.port = 22222;
@@ -59,5 +51,4 @@ in
       guest.port = 443;
     }
   ];
-  };
 }

website/default.nix

@@ -1,4 +1,4 @@
-{ sources ? import ./npins
+{ sources ? import ../npins
 , system ? builtins.currentSystem
 , pkgs ? import sources.nixpkgs {
     inherit system;
@@ -65,12 +65,12 @@ rec {
   tests = with pkgs; with lib;
     let
       source = fileset.toSource {
-        root = ./.;
+        root = ../.;
         fileset = fileset.unions [
           ./default.nix
           ./tests.nix
           ./lib.nix
-          ./npins
+          ../npins
         ];
       };
     in
@@ -86,7 +86,7 @@ rec {
         # adding it verbatim will result in <hash'>-<hash>-source, so rename it first
         cp -r ${sources.nixpkgs} source
         nix-store --add --store "$HOME" source
-        ${getExe nix-unit} --gc-roots-dir "$HOME" --store "$HOME" ${source}/tests.nix "$@"
+        ${getExe nix-unit} --gc-roots-dir "$HOME" --store "$HOME" ${source}/website/tests.nix "$@"
         touch $out
       '';
 }