Compare commits

...
Sign in to create a new pull request.

5 commits

Author SHA1 Message Date
ffd6213549 Add login indicator 2025-02-18 11:01:09 +01:00
Kiara Grouwstra
ec4e802124
add .envrc files 2025-02-13 14:48:21 +01:00
f97dc7e121 fix settings and add dummy view
This introduces customisation to `settings.py` that
- allow controlling the relevant parameters from our systemd wrapper
  (more brittle and non-obvious than it should be, see TODOs)
- correctly configure SASS processing and static file compression
  (not as easy as it sounds)
2025-02-13 00:26:28 +01:00
7c33e8aaf3 scaffold Django web service
This setup is greatly inspired by the one used for [0], although with
notable modifications, such as:
- a SASS preprocessor and CSS compressor
- more streamlined NixOS integration tests
- cleaned up service configuration
- a few notes on how to do things better in the future

[0]: https://github.com/Nix-Security-WG/nix-security-tracker/

Apart from cloning the Nix setup, there were additional steps:
- Create an empty `src` directory, since the package requires it
- In the development shell, run `django-admin startproject panel src`

Note that while you can already do

```bash
manage migrate
manage runserver
```

the NixOS integration tests will fail, since `settings.py` needs
careful massaging to expose knobs that can be turned from our systemd
wrapper. The required changes are introduced in the next commit to make
them observable.

Noteworthy related work:

- https://github.com/sephii/django.nix

  Rather mature setup with a clean interface, uses Caddy as reverse proxy.

- https://git.dgnum.eu/mdebray/djangonix

  A work-in-progress attempt to capture more moving parts through the
  module system, in particular secrets.

- https://github.com/DavHau/django-nixos

  Out of date and somewhat simplistic, but serves as a reasonable
  example for what can be done

I chose the variant I'm intimately familiar with in order to be able to
pass on knowledge or help with maintenance. But for the future
I strongly recommend picking the good bits from the other
implementations that control complexity in static configuration parts
through Nix expressions.
2025-02-13 00:26:28 +01:00
3bbd6acf4f re-use global pins 2025-02-13 00:26:28 +01:00
24 changed files with 757 additions and 5 deletions

10
.envrc Normal file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# the shebang is ignored, but nice for editors
# shellcheck shell=bash
if type -P lorri &>/dev/null; then
eval "$(lorri direnv --flake .)"
else
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
use flake
fi

1
.gitignore vendored
View file

@ -6,7 +6,6 @@ tmp/
.proxmox
/.pre-commit-config.yaml
nixos.qcow2
.envrc
.direnv
result*
.nixos-test-history

View file

@ -51,6 +51,7 @@
"keys"
"secrets"
"services"
"panel"
];
files = "^((" + concatStringsSep "|" optin + ")/.*\\.nix|[^/]*\\.nix)$";
in

10
panel/.envrc Normal file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# the shebang is ignored, but nice for editors
# shellcheck shell=bash
if type -P lorri &>/dev/null; then
eval "$(lorri direnv)"
else
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
use_nix
fi

13
panel/.gitignore vendored Normal file
View file

@ -0,0 +1,13 @@
# Nix
.direnv
result*
# Python
*.pyc
__pycache__
# Django, application-specific
db.sqlite3
src/db.sqlite3
src/static
.credentials

46
panel/README.md Normal file
View file

@ -0,0 +1,46 @@
# Fediversity Panel
The Fediversity Panel is a web service for managing Fediversity deployments with a graphical user interface, written in Django.
## Development
- To obtain all tools related to this project, enter the development environment with `nix-shell`.
If you want to do that automatically on entering this directory:
- [Set up `direnv`](https://github.com/nix-community/nix-direnv#installation)
- Run `direnv allow` in the directory where repository is stored on your machine
> **Note**
>
> This is a security boundary, and allows automatically running code from this repository on your machine.
- Run NixOS integration tests and Django unit tests:
```bash
nix-build -A tests
```
- List all available Django management commands with:
```shell-session
manage
```
- Run the server locally
```shell-session
manage runserver
```
- Whenever you add a field in the database schema, run:
```console
manage makemigrations
```
Then before starting the server again, run:
```
manage migrate
```

53
panel/default.nix Normal file
View file

@ -0,0 +1,53 @@
{
system ? builtins.currentSystem,
sources ? import ../npins,
pkgs ? import sources.nixpkgs {
inherit system;
config = { };
overlays = [ ];
},
}:
let
package =
let
callPackage = pkgs.lib.callPackageWith (pkgs // pkgs.python3.pkgs);
in
callPackage ./nix/package.nix { };
pkgs' = pkgs.extend (_final: _prev: { panel = package; });
manage = pkgs.writeScriptBin "manage" ''
exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
'';
in
{
shell = pkgs.mkShellNoCC {
inputsFrom = [ package ];
packages = [
pkgs.npins
manage
];
env = {
NPINS_DIRECTORY = toString ../npins;
};
shellHook = ''
# in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
# use this directory for testing with local secrets
mkdir -p .credentials
echo secret > ${builtins.toString ./.credentials}/SECRET_KEY
export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
export DATABASE_URL="sqlite:///${toString ./src}/db.sqlite3"
'';
};
tests = pkgs'.callPackage ./nix/tests.nix { };
inherit package;
# re-export inputs so they can be overridden granularly
# (they can't be accessed from the outside any other way)
inherit
sources
system
pkgs
;
}

199
panel/nix/configuration.nix Normal file
View file

@ -0,0 +1,199 @@
{
config,
pkgs,
lib,
...
}:
let
inherit (lib)
concatStringsSep
mapAttrsToList
mkDefault
mkEnableOption
mkIf
mkOption
mkPackageOption
optionalString
types
;
inherit (pkgs) writeShellApplication;
# TODO: configure the name globally for everywhere it's used
name = "panel";
cfg = config.services.${name};
database-url = "sqlite:////var/lib/${name}/db.sqlite3";
python-environment = pkgs.python3.withPackages (
ps: with ps; [
cfg.package
uvicorn
]
);
configFile = pkgs.concatText "configuration.py" [
((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
(builtins.toFile "extra-settings.py" cfg.extra-settings)
];
manage-service = writeShellApplication {
name = "manage";
text = ''exec ${cfg.package}/bin/manage.py "$@"'';
};
manage-admin = writeShellApplication {
# This allows running the `manage` command in the system environment, e.g. to initialise an admin user
# Executing
name = "manage";
text =
''
systemd-run --pty \
--same-dir \
--wait \
--collect \
--service-type=exec \
--unit "manage-${name}.service" \
--property "User=${name}" \
--property "Group=${name}" \
--property "Environment=DATABASE_URL=${database-url} USER_SETTINGS_FILE=${configFile}" \
''
+ optionalString (credentials != [ ]) (
(concatStringsSep " \\\n" (map (cred: "--property 'LoadCredential=${cred}'") credentials)) + " \\\n"
)
+ ''
${lib.getExe manage-service} "$@"
'';
};
credentials = mapAttrsToList (name: secretPath: "${name}:${secretPath}") cfg.secrets;
in
# TODO: for a more clever and generic way of running Django services:
# https://git.dgnum.eu/mdebray/djangonix/
# unlicensed at the time of writing, but surely worth taking some inspiration from...
{
options.services.${name} = {
enable = mkEnableOption "Service configuration for `${name}`";
# NOTE: this requires that the package is present in `pkgs`
package = mkPackageOption pkgs name { };
production = mkOption {
type = types.bool;
default = true;
};
restart = mkOption {
description = "systemd restart behavior";
type = types.enum [
"no"
"on-success"
"on-failure"
"on-abnormal"
"on-abort"
"always"
];
default = "always";
};
domain = mkOption { type = types.str; };
host = mkOption {
type = types.str;
default = "127.0.0.1";
};
port = mkOption {
type = types.port;
default = 8000;
};
settings = mkOption {
type = types.attrsOf types.anything;
default = {
STATIC_ROOT = mkDefault "/var/lib/${name}/static";
DEBUG = mkDefault false;
ALLOWED_HOSTS = mkDefault [
cfg.domain
cfg.host
"localhost"
"[::1]"
];
CSRF_TRUSTED_ORIGINS = mkDefault [ "https://${cfg.domain}" ];
COMPRESS_OFFLINE = true;
LIBSASS_OUTPUT_STYLE = "compressed";
};
description = ''
Django configuration as an attribute set.
Name-value pairs will be converted to Python variable assignments.
'';
};
extra-settings = mkOption {
type = types.lines;
default = "";
description = ''
Django configuration written in Python verbatim.
Contents will be appended to the definitions in `settings`.
'';
};
secrets = mkOption {
type = types.attrsOf types.path;
default = { };
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ manage-admin ];
services = {
nginx.enable = true;
nginx.virtualHosts = {
${cfg.domain} =
{
locations = {
"/".proxyPass = "http://localhost:${toString cfg.port}";
"/static/".alias = "/var/lib/${name}/static/";
};
}
// lib.optionalAttrs cfg.production {
enableACME = true;
forceSSL = true;
};
};
};
users.users.${name} = {
isSystemUser = true;
group = name;
};
users.groups.${name} = { };
systemd.services.${name} = {
description = "${name} ASGI server";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = [
python-environment
manage-service
];
preStart = ''
# Auto-migrate on first run or if the package has changed
versionFile="/var/lib/${name}/package-version"
if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
manage migrate --no-input
manage collectstatic --no-input --clear
manage compress --force
echo ${cfg.package} > "$versionFile"
fi
'';
script = ''
uvicorn ${name}.asgi:application --host ${cfg.host} --port ${toString cfg.port}
'';
serviceConfig = {
Restart = "always";
User = name;
WorkingDirectory = "/var/lib/${name}";
StateDirectory = name;
RuntimeDirectory = name;
LogsDirectory = name;
} // lib.optionalAttrs (credentials != [ ]) { LoadCredential = credentials; };
environment = {
USER_SETTINGS_FILE = "${configFile}";
DATABASE_URL = database-url;
};
};
};
}

57
panel/nix/package.nix Normal file
View file

@ -0,0 +1,57 @@
{
lib,
buildPythonPackage,
setuptools,
django_4,
django-compressor,
django-libsass,
dj-database-url,
}:
let
src =
with lib.fileset;
toSource {
root = ../src;
fileset = intersection (gitTracked ../../.) ../src;
};
pyproject = with lib; fromTOML pyproject-toml;
# TODO: define this globally
name = "panel";
# TODO: we may want this in a file so it's easier to read statically
version = "0.0.0";
pyproject-toml = ''
[project]
name = "Fediversity-Panel"
version = "${version}"
[tool.setuptools]
packages = [ "${name}" ]
include-package-data = true
'';
in
buildPythonPackage {
pname = name;
inherit (pyproject.project) version;
pyproject = true;
inherit src;
preBuild = ''
echo "recursive-include ${name} *" > MANIFEST.in
cp ${builtins.toFile "source" pyproject-toml} pyproject.toml
'';
propagatedBuildInputs = [
setuptools
django_4
django-compressor
django-libsass
dj-database-url
];
postInstall = ''
mkdir -p $out/bin
cp -v ${src}/manage.py $out/bin/manage.py
chmod +x $out/bin/manage.py
wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
'';
}

62
panel/nix/tests.nix Normal file
View file

@ -0,0 +1,62 @@
{ lib, pkgs }:
let
# TODO: specify project/service name globally
name = "panel";
defaults = {
services.${name} = {
enable = true;
production = false;
restart = "no";
domain = "example.com";
secrets = {
SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret";
};
};
virtualisation = {
memorySize = 2048;
cores = 2;
};
};
in
lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) {
application-tests = {
inherit defaults;
nodes.server = _: { imports = [ ./configuration.nix ]; };
# run all application-level tests managed by Django
# https://docs.djangoproject.com/en/5.0/topics/testing/overview/
testScript = ''
server.succeed("manage test")
'';
};
admin = {
inherit defaults;
nodes.server = _: { imports = [ ./configuration.nix ]; };
# check that the admin interface is served
testScript = ''
server.wait_for_unit("multi-user.target")
server.wait_for_unit("${name}.service")
server.wait_for_open_port(8000)
server.succeed("curl --fail -L -H 'Host: example.org' http://localhost/admin")
'';
};
sass-processing = {
inherit defaults;
nodes.server = _: { imports = [ ./configuration.nix ]; };
extraPythonPackages = ps: with ps; [ beautifulsoup4 ];
skipTypeCheck = true;
# check that stylesheets are pre-processed and served
testScript = ''
from bs4 import BeautifulSoup
server.wait_for_unit("multi-user.target")
server.wait_for_unit("${name}.service")
server.wait_for_open_port(8000)
stdout = server.succeed("curl --fail -H 'Host: example.org' http://localhost")
# the CSS is auto-generated with a hash in the file name
html = BeautifulSoup(stdout, 'html.parser')
css = html.find('link', type="text/css")['href']
server.succeed(f"curl --fail -H 'Host: example.org' http://localhost/{css}")
'';
};
}

1
panel/shell.nix Normal file
View file

@ -0,0 +1 @@
(import ./. { }).shell

22
panel/src/manage.py Executable file
View file

@ -0,0 +1,22 @@
#!/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/bin/python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

View file

16
panel/src/panel/asgi.py Normal file
View file

@ -0,0 +1,16 @@
"""
ASGI config for panel project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
application = get_asgi_application()

171
panel/src/panel/settings.py Normal file
View file

@ -0,0 +1,171 @@
"""
Django settings for panel project.
Generated by 'django-admin startproject' using Django 4.2.16.
For more information on this file, see
https://docs.djangoproject.com/en/4.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
import sys
import os
import importlib.util
import dj_database_url
from os import environ as env
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
def get_secret(name: str, encoding: str = "utf-8") -> str:
credentials_dir = env.get("CREDENTIALS_DIRECTORY")
if credentials_dir is None:
raise RuntimeError("No credentials directory available.")
try:
with open(f"{credentials_dir}/{name}", encoding=encoding) as f:
secret = f.read().removesuffix("\n")
except FileNotFoundError:
raise RuntimeError(f"No secret named {name} found in {credentials_dir}.")
return secret
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = get_secret("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
"panel",
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'compressor',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'panel.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'panel.wsgi.application'
# Database
# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
# https://github.com/jazzband/dj-database-url
DATABASES = {
'default': dj_database_url.config(),
}
# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.2/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.2/howto/static-files/
STATIC_URL = 'static/'
STATIC_ROOT = os.path.join(BASE_DIR, "static/")
STATICFILES_FINDERS = [
"django.contrib.staticfiles.finders.FileSystemFinder",
"django.contrib.staticfiles.finders.AppDirectoriesFinder",
"compressor.finders.CompressorFinder",
]
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
COMPRESS_PRECOMPILERS = [
("text/x-sass", "django_libsass.SassCompiler"),
]
# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# Customization via user settings
# This must be at the end, as it must be able to override the above
# TODO: we may want to do this with a flat environment instead, and get all values from `os.environ.get()`.
# this would make it more obvious which moving parts there are, if that environment is specified for development/staging/production in a visible place.
user_settings_file = env.get("USER_SETTINGS_FILE", None)
if user_settings_file is not None:
spec = importlib.util.spec_from_file_location("user_settings", user_settings_file)
if spec is None or spec.loader is None:
raise RuntimeError("User settings specification failed!")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules["user_settings"] = module
from user_settings import * # noqa: F403 # pyright: ignore [reportMissingImports]

View file

@ -0,0 +1,5 @@
body
padding: 0
margin: 0
font-family: sans-serif
box-sizing: border-box

View file

@ -0,0 +1,36 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>{% block title %}Fediversity Panel{% endblock %}</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
{% load compress %}
{% compress css %}
<link rel="stylesheet" type="text/x-sass" href="/static/style.sass" />
{% endcompress %}
{% block extra_head %}{% endblock extra_head %}
</head>
<body>
<header>
{% if user.is_authenticated %}
<p>Welcome, {{ user.username }}!</p>
{% else %}
<p>You are not logged in.</p>
{% endif %}
</header>
{% block navigation %}
<nav>
</nav>
{% endblock navigation %}
{% block layout %}
<article>
{% block content %}{% endblock content %}
</article>
{% endblock layout %}
</body>
</html>

View file

@ -0,0 +1,7 @@
{% extends "base.html" %}
{% block content %}
<h1>Fediversity Panel</h1>
<p>Hello world!</p>
{% endblock %}

24
panel/src/panel/urls.py Normal file
View file

@ -0,0 +1,24 @@
"""
URL configuration for panel project.
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from panel import views
urlpatterns = [
path('admin/', admin.site.urls),
path('', views.Index.as_view(), name='index'),
]

4
panel/src/panel/views.py Normal file
View file

@ -0,0 +1,4 @@
from django.views.generic import TemplateView
class Index(TemplateView):
template_name = 'index.html'

16
panel/src/panel/wsgi.py Normal file
View file

@ -0,0 +1,16 @@
"""
WSGI config for panel project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
application = get_wsgi_application()

View file

@ -1,4 +1,4 @@
{ sources ? import ./npins
{ sources ? import ../npins
, system ? builtins.currentSystem
, pkgs ? import sources.nixpkgs {
inherit system;
@ -65,12 +65,12 @@ rec {
tests = with pkgs; with lib;
let
source = fileset.toSource {
root = ./.;
root = ../.;
fileset = fileset.unions [
./default.nix
./tests.nix
./lib.nix
./npins
../npins
];
};
in
@ -86,7 +86,7 @@ rec {
# adding it verbatim will result in <hash'>-<hash>-source, so rename it first
cp -r ${sources.nixpkgs} source
nix-store --add --store "$HOME" source
${getExe nix-unit} --gc-roots-dir "$HOME" --store "$HOME" ${source}/tests.nix "$@"
${getExe nix-unit} --gc-roots-dir "$HOME" --store "$HOME" ${source}/website/tests.nix "$@"
touch $out
'';
}