From 6100b278b65507b65fd835cb6d6d6dd20f9cd90e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 1 May 2025 01:26:52 +0200 Subject: [PATCH] generate Python data models from module options (#285) this shows a proof of concept for generating Django forms from NixOS modules note that the form behavior is still rather clumsy and doesn't exactly map to the module semantics: - since forms can only be sent wholesale, empty form fields will show up as empty strings and break validation without additional cleanup (not done here) - it's not possible to faithfully translate `type = submodule { /* ... */}; default = {};`, since the default is translated to an empty dict `{}`. this is because the JSON schema converter does not preserve type information. this can be added by making it use `$defs` [1], but that would likely amount to half a rewrite - there's a glitch in enum default values that needs to be fixed in `datamodel-code-generator` [0] [0]: https://github.com/koxudaxi/datamodel-code-generator/blob/dd44480359c81257e55be18afefec0b3f0267ccd/src/datamodel_code_generator/parser/base.py#L1015 [1]: https://json-schema.org/understanding-json-schema/structuring#defs a generated file will be placed into the source (by the development shell and the package respectively) that declares Pydantic types from which to render the form. it looks something like this: ```python from __future__ import annotations from enum import Enum from typing import Optional from pydantic import BaseModel, Extra, Field from drf_pydantic import BaseModel class Domain(Enum): fediversity_net = 'fediversity.net' # ... class Model(BaseModel): class Config: extra = Extra.forbid domain: Optional[Domain] = Field( 'fediversity.net', description='Apex domain under which the services will be deployed.\n', ) # ... ``` --- deployment/default.nix | 11 ++ deployment/options.nix | 102 ++++++++++++++ npins/sources.json | 12 ++ panel/.gitignore | 1 + panel/nix/package.nix | 21 +++ .../python-packages/drf-pydantic/default.nix | 40 ++++++ panel/src/panel/configuration/forms.py | 117 ---------------- ...0002_alter_configuration_value_and_more.py | 5 +- panel/src/panel/models.py | 6 +- panel/src/panel/settings.py | 1 + .../panel/templates/configuration_form.html | 5 +- .../panel/tests/test_configuration_form.py | 27 ++-- panel/src/panel/urls.py | 1 - panel/src/panel/views.py | 131 +++++++----------- 14 files changed, 258 insertions(+), 222 deletions(-) create mode 100644 deployment/options.nix create mode 100644 panel/nix/python-packages/drf-pydantic/default.nix delete mode 100644 panel/src/panel/configuration/forms.py diff --git a/deployment/default.nix b/deployment/default.nix index ae7ff5f1..8db424d7 100644 --- a/deployment/default.nix +++ b/deployment/default.nix @@ -44,6 +44,17 @@ in { providers, ... }: { + options = { + deployment = lib.mkOption { + description = '' + Configuration to be deployed + ''; + # XXX(@fricklerhandwerk): + # misusing this will produce obscure errors that will be truncated by NixOps4 + type = lib.types.submodule ./options.nix; + }; + }; + config = { providers = { inherit (nixops4.modules.nixops4Provider) local; }; diff --git a/deployment/options.nix b/deployment/options.nix new file mode 100644 index 00000000..0c5fa078 --- /dev/null +++ b/deployment/options.nix @@ -0,0 +1,102 @@ +/** + Deployment options as to be presented in the front end. + + These are converted to JSON schema in order to generate front-end forms etc. + For this to work, options must not have types `functionTo` or `package`, and must not access `config` for their default values. + + The options are written in a cumbersome way because the JSON schema converter can't evaluate a submodule option's default value, which thus all must be set to `null`. + This can be fixed if we made the converter aware of [`$defs`], but that would likely amount to half a rewrite. + + [`$defs`]: https://json-schema.org/understanding-json-schema/structuring#defs +*/ +{ + lib, + ... +}: +let + inherit (lib) types mkOption; +in +{ + options = { + enable = lib.mkEnableOption "Fediversity configuration"; + domain = mkOption { + type = + with types; + enum [ + "fediversity.net" + ]; + description = '' + Apex domain under which the services will be deployed. + ''; + default = "fediversity.net"; + }; + pixelfed = mkOption { + description = '' + Configuration for the Pixelfed service + ''; + type = + with types; + nullOr (submodule { + options = { + enable = lib.mkEnableOption "Pixelfed"; + }; + }); + default = null; + }; + peertube = mkOption { + description = '' + Configuration for the PeerTube service + ''; + type = + with types; + nullOr (submodule { + options = { + enable = lib.mkEnableOption "Peertube"; + }; + }); + default = null; + }; + mastodon = mkOption { + description = '' + Configuration for the Mastodon service + ''; + type = + with types; + nullOr (submodule { + options = { + enable = lib.mkEnableOption "Mastodon"; + }; + }); + default = null; + }; + initialUser = mkOption { + description = '' + Some services require an initial user to access them. + This option sets the credentials for such an initial user. + ''; + type = + with types; + nullOr (submodule { + options = { + displayName = mkOption { + type = types.str; + description = "Display name of the user"; + }; + username = mkOption { + type = types.str; + description = "Username for login"; + }; + email = mkOption { + type = types.str; + description = "User's email address"; + }; + password = mkOption { + type = types.str; + description = "Password for login"; + }; + }; + }); + default = null; + }; + }; +} diff --git a/npins/sources.json b/npins/sources.json index ea7efb09..1c8bf296 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -1,5 +1,17 @@ { "pins": { + "clan-core": { + "type": "Git", + "repository": { + "type": "Git", + "url": "https://git.clan.lol/clan/clan-core" + }, + "branch": "main", + "submodules": false, + "revision": "ce55397ef778bc5d460b26698fed0d1eaa8f8bd6", + "url": null, + "hash": "1w2gsy6qwxa5abkv8clb435237iifndcxq0s79wihqw11a5yb938" + }, "htmx": { "type": "GitRelease", "repository": { diff --git a/panel/.gitignore b/panel/.gitignore index 5ab365e7..bcdd5369 100644 --- a/panel/.gitignore +++ b/panel/.gitignore @@ -11,4 +11,5 @@ db.sqlite3 src/db.sqlite3 src/static src/panel/static/htmx* +src/panel/configuration/schema.py .credentials diff --git a/panel/nix/package.nix b/panel/nix/package.nix index 267aafe6..e13e015a 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -2,6 +2,9 @@ lib, sqlite, python3, + python3Packages, + callPackage, + runCommand, sources ? import ../../npins, }: let @@ -25,11 +28,28 @@ let packages = [ "${name}" ] include-package-data = true ''; + generated = [ { from = "${sources.htmx}/dist/htmx.min.js"; to = "./panel/static/htmx.min.js"; } + { + from = + let + jsonschema = callPackage "${sources.clan-core}/lib/jsonschema" { } { }; + frontend-options = jsonschema.parseModule ../../deployment/options.nix; + schema = with builtins; toFile "schema.json" (toJSON frontend-options); + codegen = "${python3Packages.datamodel-code-generator}/bin/datamodel-codegen"; + pydantic = runCommand "schema.py" { } '' + # replace plain `pydantic` with `drf_pydantic` so we can create forms automatically + ${codegen} --input ${schema} | sed '/from pydantic/a\ + from drf_pydantic import BaseModel' > $out + ''; + in + "${pydantic}"; + to = "./panel/configuration/schema.py"; + } ]; in python3.pkgs.buildPythonPackage { @@ -51,6 +71,7 @@ python3.pkgs.buildPythonPackage { django-debug-toolbar django-libsass django-pydantic-field + drf-pydantic django_4 setuptools ]; diff --git a/panel/nix/python-packages/drf-pydantic/default.nix b/panel/nix/python-packages/drf-pydantic/default.nix new file mode 100644 index 00000000..2fb017a6 --- /dev/null +++ b/panel/nix/python-packages/drf-pydantic/default.nix @@ -0,0 +1,40 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + setuptools, + django, + pydantic, + hatchling, + djangorestframework, +}: + +buildPythonPackage rec { + pname = "drf-pydantic"; + version = "v2.7.1"; + pyproject = true; + + src = fetchFromGitHub { + owner = "georgebv"; + repo = pname; + rev = version; + hash = "sha256-ABtSoxj/+HHq4hj4Yb6bEiyOl00TCO/9tvBzhv6afxM="; + }; + + nativeBuildInputs = [ + setuptools + hatchling + ]; + + propagatedBuildInputs = [ + django + pydantic + djangorestframework + ]; + + meta = with lib; { + description = ""; + homepage = "https://github.com/${src.owner}/${pname}"; + license = licenses.mit; + }; +} diff --git a/panel/src/panel/configuration/forms.py b/panel/src/panel/configuration/forms.py deleted file mode 100644 index e79ecf5c..00000000 --- a/panel/src/panel/configuration/forms.py +++ /dev/null @@ -1,117 +0,0 @@ -from django import forms -from django.db import models -from pydantic import BaseModel, Field -from enum import Enum -from django_pydantic_field import SchemaField - -# TODO(@fricklerhandwerk): -# Eventually should probably maintain a separate series of configuration schema versions for each service. -# I didn't start it here yet, mainly to keep it readable. - - -class PeerTube(BaseModel): - enable: bool = Field( - default=False, - description="Enable PeerTube", - ) - - -class Pixelfed(BaseModel): - enable: bool = Field( - default=False, - description="Enable Pixelfed", - ) - - -class Mastodon(BaseModel): - enable: bool = Field( - default=False, - description="Enable Mastodon", - ) - - -class Configuration(BaseModel): - enable: bool = Field( - default=False, - description="Enable the configuration", - ) - - # XXX: hard-code available apex domains for now, - # they will be prefixed by the user name - class Domain(Enum): - NET = "fediversity.net" - - domain: Domain = Field( - default=Domain.NET, - description="DNS domain where to expose services" - ) - - peertube: PeerTube = Field( - default=PeerTube(), - description="Configuration for PeerTube", - ) - - pixelfed: Pixelfed = Field( - default=Pixelfed(), - description="Configuration for Pixelfed", - ) - - mastodon: Mastodon = Field( - default=Mastodon(), - description="Configuration for Mastodon", - ) - -# TODO@(fricklerhandwerk): -# generate this automatically from the Pydantic model so this code can go away: -# - add custom types, splicing `forms.Form` into `pydantic.BaseModel` and `forms.Field` into `pydantic.Field` so we can attach Django validators to Pydantic models -# - map primitive or well-known field types as captured by `pydantic.Field` to `forms.Field` constructors, and `BaseModel` to `Form` using `BaseModel.model_fields` -# - use a custom'{"name": "John Doe", "age": 30 }' widget that renders nested forms in e.g. a `
` -# - likely this also needs overriding `as_p()`, `as_li()` -# more inspiration: https://levelup.gitconnected.com/how-to-export-pydantic-models-as-django-forms-c1b59ddca580 -# that work goes through the JSON Schema generated by Pydantic, which seems unnecessary to me, but it follows the same princple -# TODO(@fricklerhandwerk): -# eventually we probably want to validate each field separately, -# so we should also auto-generate views per field that will be called by htmx directives defined in the templates. -# those htmx parts can be generated by each form field using `attrs` - - -class Form(forms.Form): - enable = forms.BooleanField(required=False) - domain = forms.ChoiceField( - required=False, - choices=[(d.name, d.value) for d in Configuration.Domain], - ) - peertube_enable = forms.BooleanField( - required=False, - label=Configuration.model_fields["peertube"].annotation.model_fields["enable"].description, - ) - pixelfed_enable = forms.BooleanField( - required=False, - label=Configuration.model_fields["pixelfed"].annotation.model_fields["enable"].description, - ) - mastodon_enable = forms.BooleanField( - required=False, - label=Configuration.model_fields["mastodon"].annotation.model_fields["enable"].description, - ) - - # HACK: take out nested dict fields manually - # TODO: make this generic - def __init__(self, *args, **kwargs): - initial = kwargs.pop('initial') - - initial["pixelfed_enable"] = initial.pop('pixelfed')["enable"] - initial["peertube_enable"] = initial.pop('peertube')["enable"] - initial["mastodon_enable"] = initial.pop('mastodon')["enable"] - - kwargs["initial"] = initial - - super().__init__(*args, **kwargs) - - def to_python(self): - return Configuration( - enable=self.cleaned_data['enable'], - domain=Configuration.Domain[self.cleaned_data['domain']], - peertube=PeerTube(enable=self.cleaned_data['peertube_enable']), - pixelfed=Pixelfed(enable=self.cleaned_data['pixelfed_enable']), - mastodon=Mastodon(enable=self.cleaned_data['mastodon_enable']), - ) diff --git a/panel/src/panel/migrations/0002_alter_configuration_value_and_more.py b/panel/src/panel/migrations/0002_alter_configuration_value_and_more.py index 77f90544..22ecdf31 100644 --- a/panel/src/panel/migrations/0002_alter_configuration_value_and_more.py +++ b/panel/src/panel/migrations/0002_alter_configuration_value_and_more.py @@ -2,7 +2,7 @@ from django.conf import settings from django.db import migrations, models -from panel.configuration.forms import Configuration +from panel.configuration import schema class Migration(migrations.Migration): @@ -16,7 +16,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='configuration', name='value', - field=models.JSONField(default=Configuration( - ).model_dump_json(), help_text='Stored configuration value'), + field=models.JSONField(default=schema.Model().model_dump_json(), help_text='Stored configuration value'), ), ] diff --git a/panel/src/panel/models.py b/panel/src/panel/models.py index 67426cb1..4652e183 100644 --- a/panel/src/panel/models.py +++ b/panel/src/panel/models.py @@ -1,13 +1,13 @@ from django.db import models from django.contrib.auth.models import User -from panel.configuration import forms +from panel.configuration import schema from pydantic import BaseModel def get_default_config(): - return forms.Configuration().model_dump_json() + return schema.Model().model_dump_json() class Configuration(models.Model): @@ -26,4 +26,4 @@ class Configuration(models.Model): @property def parsed_value(self) -> BaseModel: - return forms.Configuration.model_validate_json(self.value) + return schema.Model.model_validate_json(self.value) diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index 6e7f0619..bbfa753a 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -61,6 +61,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_pydantic_field', + 'rest_framework', 'debug_toolbar', 'compressor', ] diff --git a/panel/src/panel/templates/configuration_form.html b/panel/src/panel/templates/configuration_form.html index 4795d35d..ac4e68c7 100644 --- a/panel/src/panel/templates/configuration_form.html +++ b/panel/src/panel/templates/configuration_form.html @@ -1,9 +1,10 @@ {% extends "base.html" %} +{% load rest_framework %} {% block content %} -
+ {% csrf_token %} - {{ form.as_p }} + {% render_form serializer %}