generate Python data models from module options (#285)

this shows a proof of concept for generating Django forms from NixOS modules

note that the form behavior is still rather clumsy and doesn't exactly map to the module semantics:
- since forms can only be sent wholesale, empty form fields will show up as empty strings
  and break validation without additional cleanup (not done here)
- it's not possible to faithfully translate `type = submodule { /* ... */}; default = {};`, since the default
  is translated to an empty dict `{}`. this is because the JSON schema converter does not preserve type information.
  this can be added by making it use `$defs` [1], but that would likely amount to half a rewrite
- there's a glitch in enum default values that needs to be fixed in `datamodel-code-generator` [0]

[0]: dd44480359/src/datamodel_code_generator/parser/base.py (L1015)
[1]: https://json-schema.org/understanding-json-schema/structuring#defs

a generated file will be placed into the source (by the development shell and the package respectively)
that declares Pydantic types from which to render the form. it looks something like this:

```python
from __future__ import annotations

from enum import Enum
from typing import Optional

from pydantic import BaseModel, Extra, Field
from drf_pydantic import BaseModel

class Domain(Enum):
    fediversity_net = 'fediversity.net'

# ...

class Model(BaseModel):
    class Config:
        extra = Extra.forbid

    domain: Optional[Domain] = Field(
        'fediversity.net',
        description='Apex domain under which the services will be deployed.\n',
    )

  # ...
```

deployment/default.nix

@@ -44,6 +44,17 @@ in
 { providers, ... }:
 
 {
+  options = {
+    deployment = lib.mkOption {
+      description = ''
+        Configuration to be deployed
+      '';
+      # XXX(@fricklerhandwerk):
+      #     misusing this will produce obscure errors that will be truncated by NixOps4
+      type = lib.types.submodule ./options.nix;
+    };
+  };
+
   config = {
     providers = { inherit (nixops4.modules.nixops4Provider) local; };
 

deployment/options.nix

@@ -0,0 +1,102 @@
+/**
+  Deployment options as to be presented in the front end.
+
+  These are converted to JSON schema in order to generate front-end forms etc.
+  For this to work, options must not have types `functionTo` or `package`, and must not access `config` for their default values.
+
+  The options are written in a cumbersome way because the JSON schema converter can't evaluate a submodule option's default value, which thus all must be set to `null`.
+  This can be fixed if we made the converter aware of [`$defs`], but that would likely amount to half a rewrite.
+
+  [`$defs`]: https://json-schema.org/understanding-json-schema/structuring#defs
+*/
+{
+  lib,
+  ...
+}:
+let
+  inherit (lib) types mkOption;
+in
+{
+  options = {
+    enable = lib.mkEnableOption "Fediversity configuration";
+    domain = mkOption {
+      type =
+        with types;
+        enum [
+          "fediversity.net"
+        ];
+      description = ''
+        Apex domain under which the services will be deployed.
+      '';
+      default = "fediversity.net";
+    };
+    pixelfed = mkOption {
+      description = ''
+        Configuration for the Pixelfed service
+      '';
+      type =
+        with types;
+        nullOr (submodule {
+          options = {
+            enable = lib.mkEnableOption "Pixelfed";
+          };
+        });
+      default = null;
+    };
+    peertube = mkOption {
+      description = ''
+        Configuration for the PeerTube service
+      '';
+      type =
+        with types;
+        nullOr (submodule {
+          options = {
+            enable = lib.mkEnableOption "Peertube";
+          };
+        });
+      default = null;
+    };
+    mastodon = mkOption {
+      description = ''
+        Configuration for the Mastodon service
+      '';
+      type =
+        with types;
+        nullOr (submodule {
+          options = {
+            enable = lib.mkEnableOption "Mastodon";
+          };
+        });
+      default = null;
+    };
+    initialUser = mkOption {
+      description = ''
+        Some services require an initial user to access them.
+        This option sets the credentials for such an initial user.
+      '';
+      type =
+        with types;
+        nullOr (submodule {
+          options = {
+            displayName = mkOption {
+              type = types.str;
+              description = "Display name of the user";
+            };
+            username = mkOption {
+              type = types.str;
+              description = "Username for login";
+            };
+            email = mkOption {
+              type = types.str;
+              description = "User's email address";
+            };
+            password = mkOption {
+              type = types.str;
+              description = "Password for login";
+            };
+          };
+        });
+      default = null;
+    };
+  };
+}

npins/sources.json

@@ -1,5 +1,17 @@
 {
   "pins": {
+    "clan-core": {
+      "type": "Git",
+      "repository": {
+        "type": "Git",
+        "url": "https://git.clan.lol/clan/clan-core"
+      },
+      "branch": "main",
+      "submodules": false,
+      "revision": "ce55397ef778bc5d460b26698fed0d1eaa8f8bd6",
+      "url": null,
+      "hash": "1w2gsy6qwxa5abkv8clb435237iifndcxq0s79wihqw11a5yb938"
+    },
     "htmx": {
       "type": "GitRelease",
       "repository": {

panel/.gitignore

@@ -11,4 +11,5 @@ db.sqlite3
 src/db.sqlite3
 src/static
 src/panel/static/htmx*
+src/panel/configuration/schema.py
 .credentials

panel/nix/package.nix

@@ -2,6 +2,9 @@
   lib,
   sqlite,
   python3,
+  python3Packages,
+  callPackage,
+  runCommand,
   sources ? import ../../npins,
 }:
 let
@@ -25,11 +28,28 @@ let
     packages = [ "${name}" ]
     include-package-data = true
   '';
+
   generated = [
     {
       from = "${sources.htmx}/dist/htmx.min.js";
       to = "./panel/static/htmx.min.js";
     }
+    {
+      from =
+        let
+          jsonschema = callPackage "${sources.clan-core}/lib/jsonschema" { } { };
+          frontend-options = jsonschema.parseModule ../../deployment/options.nix;
+          schema = with builtins; toFile "schema.json" (toJSON frontend-options);
+          codegen = "${python3Packages.datamodel-code-generator}/bin/datamodel-codegen";
+          pydantic = runCommand "schema.py" { } ''
+            # replace plain `pydantic` with `drf_pydantic` so we can create forms automatically
+            ${codegen} --input ${schema} | sed '/from pydantic/a\
+            from drf_pydantic import BaseModel' > $out
+          '';
+        in
+        "${pydantic}";
+      to = "./panel/configuration/schema.py";
+    }
   ];
 in
 python3.pkgs.buildPythonPackage {
@@ -51,6 +71,7 @@ python3.pkgs.buildPythonPackage {
         django-debug-toolbar
         django-libsass
         django-pydantic-field
+        drf-pydantic
         django_4
         setuptools
       ];

panel/nix/python-packages/drf-pydantic/default.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools,
+  django,
+  pydantic,
+  hatchling,
+  djangorestframework,
+}:
+
+buildPythonPackage rec {
+  pname = "drf-pydantic";
+  version = "v2.7.1";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "georgebv";
+    repo = pname;
+    rev = version;
+    hash = "sha256-ABtSoxj/+HHq4hj4Yb6bEiyOl00TCO/9tvBzhv6afxM=";
+  };
+
+  nativeBuildInputs = [
+    setuptools
+    hatchling
+  ];
+
+  propagatedBuildInputs = [
+    django
+    pydantic
+    djangorestframework
+  ];
+
+  meta = with lib; {
+    description = "";
+    homepage = "https://github.com/${src.owner}/${pname}";
+    license = licenses.mit;
+  };
+}

panel/src/panel/configuration/forms.py

@@ -1,117 +0,0 @@
-from django import forms
-from django.db import models
-from pydantic import BaseModel, Field
-from enum import Enum
-from django_pydantic_field import SchemaField
-
-# TODO(@fricklerhandwerk):
-#     Eventually should probably maintain a separate series of configuration schema versions for each service.
-#     I didn't start it here yet, mainly to keep it readable.
-
-
-class PeerTube(BaseModel):
-    enable: bool = Field(
-        default=False,
-        description="Enable PeerTube",
-    )
-
-
-class Pixelfed(BaseModel):
-    enable: bool = Field(
-        default=False,
-        description="Enable Pixelfed",
-    )
-
-
-class Mastodon(BaseModel):
-    enable: bool = Field(
-        default=False,
-        description="Enable Mastodon",
-    )
-
-
-class Configuration(BaseModel):
-    enable: bool = Field(
-        default=False,
-        description="Enable the configuration",
-    )
-
-    # XXX: hard-code available apex domains for now,
-    #      they will be prefixed by the user name
-    class Domain(Enum):
-        NET = "fediversity.net"
-
-    domain: Domain = Field(
-        default=Domain.NET,
-        description="DNS domain where to expose services"
-    )
-
-    peertube: PeerTube = Field(
-        default=PeerTube(),
-        description="Configuration for PeerTube",
-    )
-
-    pixelfed: Pixelfed = Field(
-        default=Pixelfed(),
-        description="Configuration for Pixelfed",
-    )
-
-    mastodon: Mastodon = Field(
-        default=Mastodon(),
-        description="Configuration for Mastodon",
-    )
-
-# TODO@(fricklerhandwerk):
-#     generate this automatically from the Pydantic model so this code can go away:
-#     - add custom types, splicing `forms.Form` into `pydantic.BaseModel` and `forms.Field` into `pydantic.Field` so we can attach Django validators to Pydantic models
-#     - map primitive or well-known field types as captured by `pydantic.Field` to `forms.Field` constructors, and `BaseModel` to `Form` using `BaseModel.model_fields`
-#     - use a custom'{"name": "John Doe",     "age": 30 }' widget that renders nested forms in e.g. a `<div>`
-#       - likely this also needs overriding `as_p()`, `as_li()`
-#     more inspiration: https://levelup.gitconnected.com/how-to-export-pydantic-models-as-django-forms-c1b59ddca580
-#     that work goes through the JSON Schema generated by Pydantic, which seems unnecessary to me, but it follows the same princple
-# TODO(@fricklerhandwerk):
-#     eventually we probably want to validate each field separately,
-#     so we should also auto-generate views per field that will be called by htmx directives defined in the templates.
-#     those htmx parts can be generated by each form field using `attrs`
-
-
-class Form(forms.Form):
-    enable = forms.BooleanField(required=False)
-    domain = forms.ChoiceField(
-        required=False,
-        choices=[(d.name, d.value) for d in Configuration.Domain],
-    )
-    peertube_enable = forms.BooleanField(
-        required=False,
-        label=Configuration.model_fields["peertube"].annotation.model_fields["enable"].description,
-    )
-    pixelfed_enable = forms.BooleanField(
-        required=False,
-        label=Configuration.model_fields["pixelfed"].annotation.model_fields["enable"].description,
-    )
-    mastodon_enable = forms.BooleanField(
-        required=False,
-        label=Configuration.model_fields["mastodon"].annotation.model_fields["enable"].description,
-    )
-
-    # HACK: take out nested dict fields manually
-    # TODO: make this generic
-    def __init__(self, *args, **kwargs):
-        initial = kwargs.pop('initial')
-
-        initial["pixelfed_enable"] = initial.pop('pixelfed')["enable"]
-        initial["peertube_enable"] = initial.pop('peertube')["enable"]
-        initial["mastodon_enable"] = initial.pop('mastodon')["enable"]
-
-        kwargs["initial"] = initial
-
-        super().__init__(*args, **kwargs)
-
-    def to_python(self):
-        return Configuration(
-            enable=self.cleaned_data['enable'],
-            domain=Configuration.Domain[self.cleaned_data['domain']],
-            peertube=PeerTube(enable=self.cleaned_data['peertube_enable']),
-            pixelfed=Pixelfed(enable=self.cleaned_data['pixelfed_enable']),
-            mastodon=Mastodon(enable=self.cleaned_data['mastodon_enable']),
-        )

panel/src/panel/migrations/0002_alter_configuration_value_and_more.py

@@ -2,7 +2,7 @@
 
 from django.conf import settings
 from django.db import migrations, models
-from panel.configuration.forms import Configuration
+from panel.configuration import schema
 
 
 class Migration(migrations.Migration):
@@ -16,7 +16,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='configuration',
             name='value',
-            field=models.JSONField(default=Configuration(
+            field=models.JSONField(default=schema.Model().model_dump_json(), help_text='Stored configuration value'),
-            ).model_dump_json(), help_text='Stored configuration value'),
         ),
     ]

panel/src/panel/models.py

@@ -1,13 +1,13 @@
 from django.db import models
 from django.contrib.auth.models import User
 
-from panel.configuration import forms
+from panel.configuration import schema
 
 from pydantic import BaseModel
 
 
 def get_default_config():
-    return forms.Configuration().model_dump_json()
+    return schema.Model().model_dump_json()
 
 
 class Configuration(models.Model):
@@ -26,4 +26,4 @@ class Configuration(models.Model):
 
     @property
     def parsed_value(self) -> BaseModel:
-        return forms.Configuration.model_validate_json(self.value)
+        return schema.Model.model_validate_json(self.value)

panel/src/panel/settings.py

@@ -61,6 +61,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django_pydantic_field',
+    'rest_framework',
     'debug_toolbar',
     'compressor',
 ]

panel/src/panel/templates/configuration_form.html

@@ -1,9 +1,10 @@
 {% extends "base.html" %}
+{% load rest_framework %}
 {% block content %}
-<form method="post" enctype="multipart/form-data" action="{% url 'save' %}">
+<form method="post" enctype="multipart/form-data" action="{% url 'configuration_form' %}">
   {% csrf_token %}
 
-  {{ form.as_p }}
+  {% render_form serializer %}
   <button id="deploy-button" class="button"
           hx-post="{% url 'deployment_status' %}"
           hx-trigger="click"

panel/src/panel/tests/test_configuration_form.py

@@ -13,7 +13,7 @@ class ConfigurationForm(TestCase):
             password=self.password
         )
 
-        self.config_url = reverse('save')
+        self.config_url = reverse('configuration_form')
 
     def test_configuration_form_submission(self):
         config = Configuration.objects.create(
@@ -21,29 +21,26 @@ class ConfigurationForm(TestCase):
         )
 
         self.client.login(username=self.username, password=self.password)
-        response = self.client.get(self.config_url)
+        response = self.client.get(self.config_url, follow=True)
         self.assertEqual(response.status_code, 200)
 
         context = response.context[0]
 
         # configuration should be disabled by default
-        self.assertFalse(context['view'].get_object().parsed_value.enable)
+        self.assertFalse(context['serializer'].instance.enable)
-        # ...and be displayed as such
-        self.assertFalse(context['form'].initial["enable"])
 
-        form_data = context['form'].initial.copy()
+        form_data = context['serializer'].instance.dict().copy()
-        form_data.update(
+        form_data = {
-            enable=True,
+            "enable": True,
-            mastodon_enable=True,
+            "mastodon.enable": True,
-        )
+        }
-        print(form_data)
+        response = self.client.post(self.config_url, data=form_data, follow=True)
-        response = self.client.post(self.config_url, data=form_data)
 
-        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response.status_code, 200)
         config.refresh_from_db()
 
-        print(config.parsed_value)
         self.assertTrue(config.parsed_value.enable)
         self.assertTrue(config.parsed_value.mastodon.enable)
         # this should not have changed
-        self.assertFalse(config.parsed_value.peertube.enable)
+        self.assertIsNone(config.parsed_value.peertube)
+        self.assertIsNone(config.parsed_value.pixelfed)

panel/src/panel/urls.py

@@ -27,5 +27,4 @@ urlpatterns = [
     path("services/", views.ServiceList.as_view(), name='service_list'),
     path("configuration/", views.ConfigurationForm.as_view(), name='configuration_form'),
     path("deployment/status/", views.DeploymentStatus.as_view(), name='deployment_status'),
-    path("save/", views.Save.as_view(), name='save'),
 ]

panel/src/panel/views.py

@@ -8,11 +8,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.auth.models import User
 from django.views.generic import TemplateView, DetailView
 from django.views.generic.edit import FormView
+from django.shortcuts import render, redirect
+
+from rest_framework.renderers import TemplateHTMLRenderer
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from pydantic import BaseModel
 
-from django.shortcuts import render
 from panel import models, settings
-from panel.configuration import forms
+from panel.configuration import schema
-
 
 class Index(TemplateView):
     template_name = 'index.html'
@@ -30,100 +35,66 @@ class ServiceList(TemplateView):
     template_name = 'service_list.html'
 
 
-class ConfigurationForm(LoginRequiredMixin, FormView):
+class ConfigurationForm(LoginRequiredMixin, APIView):
+    renderer_classes = [TemplateHTMLRenderer]
     template_name = 'configuration_form.html'
     success_url = reverse_lazy('configuration_form')
-    form_class = forms.Form
 
     def get_object(self):
-        """Get or create the configuration object for the current user"""
         obj, created = models.Configuration.objects.get_or_create(
             operator=self.request.user,
         )
-
         return obj
 
-    # TODO(@fricklerhandwerk):
+    def get(self, request):
-    #     this should probably live somewhere else
-    def convert_enums_to_names(self, data_dict):
-        """
-        Recursively convert all enum values in a dictionary to their string names.
-        This handles nested dictionaries and lists as well.
-
-        Needed for converting a Pydantic `BaseModel` instance to a `Form` input.
-        """
-        if isinstance(data_dict, dict):
-            result = {}
-            for key, value in data_dict.items():
-                if isinstance(value, Enum):
-                    # Convert Enum to its name
-                    result[key] = value.name
-                elif isinstance(value, (dict, list)):
-                    # Recursively process nested structures
-                    result[key] = self.convert_enums_to_names(value)
-                else:
-                    # Keep other values as is
-                    result[key] = value
-            return result
-        elif isinstance(data_dict, list):
-            # Process each item in the list
-            return [self.convert_enums_to_names(item) for item in data_dict]
-        elif isinstance(data_dict, Enum):
-            # Convert single Enum value
-            return data_dict.name
-        else:
-            # Return non-dict, non-list, non-Enum values as is
-            return data_dict
-
-    def get_initial(self):
-        initial = super().get_initial()
         config = self.get_object()
-        config_dict = config.parsed_value.model_dump()
+        serializer = schema.Model.drf_serializer(instance=config.parsed_value)
+        return Response({'serializer': serializer})
 
-        initial.update(self.convert_enums_to_names(config_dict))
+    def post(self, request):
-        return initial
+        config = self.get_object()
+        serializer = schema.Model.drf_serializer(
+            instance=config.parsed_value,
+            data=request.data
+        )
 
+        if not serializer.is_valid():
+            return Response({'serializer': serializer})
 
-class Save(ConfigurationForm):
+        config.value = json.dumps(serializer.validated_data)
-    def form_valid(self, form):
+        config.save()
-        obj = self.get_object()
+        return redirect(self.success_url)
-        obj.value = form.to_python().model_dump_json()
-        obj.save()
-
-        return super().form_valid(form)
-
 
+# TODO(@fricklerhandwerk):
+#     this is broken after changing the form view.
+#     but if there's no test for it, how do we know it ever worked in the first place?
 class DeploymentStatus(ConfigurationForm):
-    def form_valid(self, form):
-        obj = self.get_object()
-        obj.value = form.to_python().model_dump_json()
-        obj.save()
 
-        # Check for deploy button
+    def post(self, request):
-        if "deploy" in self.request.POST.keys():
+        config = self.get_object()
-            deployment_result, deployment_params = self.deployment(obj)
+        serializer = schema.Model.drf_serializer(
-            deployment_succeeded = deployment_result.returncode == 0
+            instance=config.parsed_value,
+            data=request.data
+        )
+
+        if not serializer.is_valid():
+            return Response({'serializer': serializer})
+
+        config.value = json.dumps(serializer.validated_data)
+        config.save()
+
+        deployment_result, deployment_params = self.deployment(config.parsed_value)
+        if deployment_result.returncode == 0:
+            deployment_status = "Deployment Succeeded"
+        else:
+            deployment_status = "Deployment Failed"
 
         return render(self.request, "partials/deployment_result.html", {
-            "deployment_succeeded": deployment_succeeded,
+            "deployment_status": deployment_status,
-            "services": {
+            "services": deployment_params.json(),
-                "peertube": {
-                    "enable": deployment_params['peertube']['enable'],
-                    "url": f"https://peertube.{deployment_params['domain']}",
-                    },
-                "pixelfed":{
-                    "enable": deployment_params['pixelfed']['enable'],
-                    "url": f"https://pixelfed.{deployment_params['domain']}",
-                    },
-                "mastodon": {
-                    "enable": deployment_params['mastodon']['enable'],
-                    "url": f"https://mastodon.{deployment_params['domain']}",
-                    },
-                },
             })
 
-    def deployment(self, obj):
+    def deployment(self, config: BaseModel):
-        submission = obj.parsed_value.model_dump_json()
         # FIXME: let the user specify these from the form (#190)
         dummy_user = {
             "initialUser": {
@@ -133,12 +104,10 @@ class DeploymentStatus(ConfigurationForm):
                 "email": "test@test.com",
             },
         }
-        # serialize back and forth now we still need to manually inject the dummy user
-        deployment_params = json.dumps(dummy_user | json.loads(submission))
         env = {
             "PATH": settings.bin_path,
             # pass in form info to our deployment
-            "DEPLOYMENT": deployment_params,
+            "DEPLOYMENT": config.json()
         }
         cmd = [
             "nix",
@@ -155,4 +124,4 @@ class DeploymentStatus(ConfigurationForm):
             cwd=settings.repo_dir,
             env=env,
         )
-        return deployment_result, json.loads(deployment_params)
+        return deployment_result, config