scaffold Django web service

This setup is greatly inspired by the one used for [0], although with
notable modifications, such as:
- a SASS preprocessor and CSS compressor
- more streamlined NixOS integration tests
- cleaned up service configuration
- a few notes on how to do things better in the future

[0]: https://github.com/Nix-Security-WG/nix-security-tracker/

Apart from cloning the Nix setup, there were additional steps:
- Create an empty `src` directory, since the package requires it
- In the development shell, run `django-admin startproject panel src`

Note that while you can already do

```bash
manage migrate
manage runserver
```

the NixOS integration tests will fail, since `settings.py` needs
careful massaging to expose knobs that can be turned from our systemd
wrapper. The required changes are introduced in the next commit to make
them observable.

Noteworthy related work:

- https://github.com/sephii/django.nix

  Rather mature setup with a clean interface, uses Caddy as reverse proxy.

- https://git.dgnum.eu/mdebray/djangonix

  A work-in-progress attempt to capture more moving parts through the
  module system, in particular secrets.

- https://github.com/DavHau/django-nixos

  Out of date and somewhat simplistic, but serves as a reasonable
  example for what can be done

I chose the variant I'm intimately familiar with in order to be able to
pass on knowledge or help with maintenance. But for the future
I strongly recommend picking the good bits from the other
implementations that control complexity in static configuration parts
through Nix expressions.

flake.nix

@@ -51,6 +51,7 @@
                 "keys"
                 "secrets"
                 "services"
+                "panel"
               ];
               files = "^((" + concatStringsSep "|" optin + ")/.*\\.nix|[^/]*\\.nix)$";
             in

panel/.gitignore

@@ -0,0 +1,13 @@
+# Nix
+.direnv
+result*
+
+# Python
+*.pyc
+__pycache__
+
+# Django, application-specific
+db.sqlite3
+src/db.sqlite3
+src/static
+.credentials

panel/README.md

@@ -0,0 +1,46 @@
+# Fediversity Panel
+
+The Fediversity Panel is a web service for managing Fediversity deployments with a graphical user interface, written in Django.
+
+## Development
+
+- To obtain all tools related to this project, enter the development environment with `nix-shell`.
+
+  If you want to do that automatically on entering this directory:
+
+  - [Set up `direnv`](https://github.com/nix-community/nix-direnv#installation)
+  - Run `direnv allow` in the directory where repository is stored on your machine
+
+    > **Note**
+    >
+    > This is a security boundary, and allows automatically running code from this repository on your machine.
+
+- Run NixOS integration tests and Django unit tests:
+
+  ```bash
+  nix-build -A tests
+  ```
+
+- List all available Django management commands with:
+
+  ```shell-session
+  manage
+  ```
+
+- Run the server locally
+
+  ```shell-session
+  manage runserver
+  ```
+
+- Whenever you add a field in the database schema, run:
+
+  ```console
+  manage makemigrations
+  ```
+
+  Then before starting the server again, run:
+
+  ```
+  manage migrate
+  ```

panel/default.nix

@@ -0,0 +1,53 @@
+{
+  system ? builtins.currentSystem,
+  sources ? import ../npins,
+  pkgs ? import sources.nixpkgs {
+    inherit system;
+    config = { };
+    overlays = [ ];
+  },
+}:
+let
+  package =
+    let
+      callPackage = pkgs.lib.callPackageWith (pkgs // pkgs.python3.pkgs);
+    in
+    callPackage ./nix/package.nix { };
+
+  pkgs' = pkgs.extend (_final: _prev: { panel = package; });
+
+  manage = pkgs.writeScriptBin "manage" ''
+    exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
+  '';
+in
+{
+  shell = pkgs.mkShellNoCC {
+    inputsFrom = [ package ];
+    packages = [
+      pkgs.npins
+      manage
+    ];
+    env = {
+      NPINS_DIRECTORY = toString ../npins;
+    };
+    shellHook = ''
+      # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
+      # use this directory for testing with local secrets
+      mkdir -p .credentials
+      echo secret > ${builtins.toString ./.credentials}/SECRET_KEY
+      export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
+      export DATABASE_URL="sqlite:///${toString ./src}/db.sqlite3"
+    '';
+  };
+
+  tests = pkgs'.callPackage ./nix/tests.nix { };
+  inherit package;
+
+  # re-export inputs so they can be overridden granularly
+  # (they can't be accessed from the outside any other way)
+  inherit
+    sources
+    system
+    pkgs
+    ;
+}

panel/nix/configuration.nix

@@ -0,0 +1,199 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    concatStringsSep
+    mapAttrsToList
+    mkDefault
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    optionalString
+    types
+    ;
+  inherit (pkgs) writeShellApplication;
+
+  # TODO: configure the name globally for everywhere it's used
+  name = "panel";
+
+  cfg = config.services.${name};
+
+  database-url = "sqlite:////var/lib/${name}/db.sqlite3";
+
+  python-environment = pkgs.python3.withPackages (
+    ps: with ps; [
+      cfg.package
+      uvicorn
+    ]
+  );
+
+  configFile = pkgs.concatText "configuration.py" [
+    ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
+    (builtins.toFile "extra-settings.py" cfg.extra-settings)
+  ];
+
+  manage-service = writeShellApplication {
+    name = "manage";
+    text = ''exec ${cfg.package}/bin/manage.py "$@"'';
+  };
+
+  manage-admin = writeShellApplication {
+    # This allows running the `manage` command in the system environment, e.g. to initialise an admin user
+    # Executing
+    name = "manage";
+    text =
+      ''
+        systemd-run --pty \
+          --same-dir \
+          --wait \
+          --collect \
+          --service-type=exec \
+          --unit "manage-${name}.service" \
+          --property "User=${name}" \
+          --property "Group=${name}" \
+          --property "Environment=DATABASE_URL=${database-url} USER_SETTINGS_FILE=${configFile}" \
+      ''
+      + optionalString (credentials != [ ]) (
+        (concatStringsSep " \\\n" (map (cred: "--property 'LoadCredential=${cred}'") credentials)) + " \\\n"
+      )
+      + ''
+        ${lib.getExe manage-service} "$@"
+      '';
+  };
+
+  credentials = mapAttrsToList (name: secretPath: "${name}:${secretPath}") cfg.secrets;
+in
+# TODO: for a more clever and generic way of running Django services:
+#       https://git.dgnum.eu/mdebray/djangonix/
+#       unlicensed at the time of writing, but surely worth taking some inspiration from...
+{
+  options.services.${name} = {
+    enable = mkEnableOption "Service configuration for `${name}`";
+    # NOTE: this requires that the package is present in `pkgs`
+    package = mkPackageOption pkgs name { };
+    production = mkOption {
+      type = types.bool;
+      default = true;
+    };
+    restart = mkOption {
+      description = "systemd restart behavior";
+      type = types.enum [
+        "no"
+        "on-success"
+        "on-failure"
+        "on-abnormal"
+        "on-abort"
+        "always"
+      ];
+      default = "always";
+    };
+    domain = mkOption { type = types.str; };
+    host = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+    };
+    port = mkOption {
+      type = types.port;
+      default = 8000;
+    };
+    settings = mkOption {
+      type = types.attrsOf types.anything;
+      default = {
+        STATIC_ROOT = mkDefault "/var/lib/${name}/static";
+        DEBUG = mkDefault false;
+        ALLOWED_HOSTS = mkDefault [
+          cfg.domain
+          cfg.host
+          "localhost"
+          "[::1]"
+        ];
+        CSRF_TRUSTED_ORIGINS = mkDefault [ "https://${cfg.domain}" ];
+        COMPRESS_OFFLINE = true;
+        LIBSASS_OUTPUT_STYLE = "compressed";
+      };
+      description = ''
+        Django configuration as an attribute set.
+        Name-value pairs will be converted to Python variable assignments.
+      '';
+    };
+    extra-settings = mkOption {
+      type = types.lines;
+      default = "";
+      description = ''
+        Django configuration written in Python verbatim.
+        Contents will be appended to the definitions in `settings`.
+      '';
+    };
+    secrets = mkOption {
+      type = types.attrsOf types.path;
+      default = { };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ manage-admin ];
+
+    services = {
+      nginx.enable = true;
+      nginx.virtualHosts = {
+        ${cfg.domain} =
+          {
+            locations = {
+              "/".proxyPass = "http://localhost:${toString cfg.port}";
+              "/static/".alias = "/var/lib/${name}/static/";
+            };
+          }
+          // lib.optionalAttrs cfg.production {
+            enableACME = true;
+            forceSSL = true;
+          };
+      };
+    };
+
+    users.users.${name} = {
+      isSystemUser = true;
+      group = name;
+    };
+
+    users.groups.${name} = { };
+    systemd.services.${name} = {
+      description = "${name} ASGI server";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [
+        python-environment
+        manage-service
+      ];
+      preStart = ''
+        # Auto-migrate on first run or if the package has changed
+        versionFile="/var/lib/${name}/package-version"
+        if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
+          manage migrate --no-input
+          manage collectstatic --no-input --clear
+          manage compress --force
+          echo ${cfg.package} > "$versionFile"
+        fi
+      '';
+      script = ''
+        uvicorn ${name}.asgi:application --host ${cfg.host} --port ${toString cfg.port}
+      '';
+      serviceConfig = {
+        Restart = "always";
+        User = name;
+        WorkingDirectory = "/var/lib/${name}";
+        StateDirectory = name;
+        RuntimeDirectory = name;
+        LogsDirectory = name;
+      } // lib.optionalAttrs (credentials != [ ]) { LoadCredential = credentials; };
+      environment = {
+        USER_SETTINGS_FILE = "${configFile}";
+        DATABASE_URL = database-url;
+      };
+    };
+  };
+}

panel/nix/package.nix

@@ -0,0 +1,57 @@
+{
+  lib,
+  buildPythonPackage,
+  setuptools,
+  django_4,
+  django-compressor,
+  django-libsass,
+  dj-database-url,
+}:
+let
+  src =
+    with lib.fileset;
+    toSource {
+      root = ../src;
+      fileset = intersection (gitTracked ../../.) ../src;
+    };
+  pyproject = with lib; fromTOML pyproject-toml;
+  # TODO: define this globally
+  name = "panel";
+  # TODO: we may want this in a file so it's easier to read statically
+  version = "0.0.0";
+  pyproject-toml = ''
+    [project]
+    name = "Fediversity-Panel"
+    version = "${version}"
+
+    [tool.setuptools]
+    packages = [ "${name}" ]
+    include-package-data = true
+  '';
+in
+buildPythonPackage {
+  pname = name;
+  inherit (pyproject.project) version;
+  pyproject = true;
+  inherit src;
+
+  preBuild = ''
+    echo "recursive-include ${name} *" > MANIFEST.in
+    cp ${builtins.toFile "source" pyproject-toml} pyproject.toml
+  '';
+
+  propagatedBuildInputs = [
+    setuptools
+    django_4
+    django-compressor
+    django-libsass
+    dj-database-url
+  ];
+
+  postInstall = ''
+    mkdir -p $out/bin
+    cp -v ${src}/manage.py $out/bin/manage.py
+    chmod +x $out/bin/manage.py
+    wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
+  '';
+}

panel/nix/tests.nix

@@ -0,0 +1,62 @@
+{ lib, pkgs }:
+let
+  # TODO: specify project/service name globally
+  name = "panel";
+  defaults = {
+    services.${name} = {
+      enable = true;
+      production = false;
+      restart = "no";
+      domain = "example.com";
+      secrets = {
+        SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret";
+      };
+    };
+
+    virtualisation = {
+      memorySize = 2048;
+      cores = 2;
+    };
+  };
+in
+lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) {
+  application-tests = {
+    inherit defaults;
+    nodes.server = _: { imports = [ ./configuration.nix ]; };
+    # run all application-level tests managed by Django
+    # https://docs.djangoproject.com/en/5.0/topics/testing/overview/
+    testScript = ''
+      server.succeed("manage test")
+    '';
+  };
+  admin = {
+    inherit defaults;
+    nodes.server = _: { imports = [ ./configuration.nix ]; };
+    # check that the admin interface is served
+    testScript = ''
+      server.wait_for_unit("multi-user.target")
+      server.wait_for_unit("${name}.service")
+      server.wait_for_open_port(8000)
+      server.succeed("curl --fail -L -H 'Host: example.org' http://localhost/admin")
+    '';
+  };
+
+  sass-processing = {
+    inherit defaults;
+    nodes.server = _: { imports = [ ./configuration.nix ]; };
+    extraPythonPackages = ps: with ps; [ beautifulsoup4 ];
+    skipTypeCheck = true;
+    # check that stylesheets are pre-processed and served
+    testScript = ''
+      from bs4 import BeautifulSoup
+      server.wait_for_unit("multi-user.target")
+      server.wait_for_unit("${name}.service")
+      server.wait_for_open_port(8000)
+      stdout = server.succeed("curl --fail -H 'Host: example.org' http://localhost")
+      # the CSS is auto-generated with a hash in the file name
+      html = BeautifulSoup(stdout, 'html.parser')
+      css = html.find('link', type="text/css")['href']
+      server.succeed(f"curl --fail -H 'Host: example.org' http://localhost/{css}")
+    '';
+  };
+}

panel/shell.nix

@@ -0,0 +1 @@
+(import ./. { }).shell

panel/src/manage.py

@@ -0,0 +1,22 @@
+#!/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/bin/python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

panel/src/panel/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for panel project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
+
+application = get_asgi_application()

panel/src/panel/settings.py

@@ -0,0 +1,123 @@
+"""
+Django settings for panel project.
+
+Generated by 'django-admin startproject' using Django 4.2.16.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.2/ref/settings/
+"""
+
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'django-insecure-9*i_k2o@x-c7w%o!*@b88t%n)eh=c2nj2f2m*-=$gwfn#zoso7'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'panel.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'panel.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': BASE_DIR / 'db.sqlite3',
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/4.2/howto/static-files/
+
+STATIC_URL = 'static/'
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

panel/src/panel/urls.py

@@ -0,0 +1,22 @@
+"""
+URL configuration for panel project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/4.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+]

panel/src/panel/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for panel project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
+
+application = get_wsgi_application()