From 0a11abe1b3264d288fe36fece794b6b765b34744 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 31 Mar 2025 09:41:13 +0200
Subject: [PATCH] POC: generate Pydantic models from NixOS modules
---
deployment/default.nix | 15 +-
deployment/options.nix | 54 +++
npins/sources.json | 15 +-
panel/.gitignore | 1 +
panel/default.nix | 17 +
panel/jsonschema.nix | 430 ++++++++++++++++++
panel/nix/package.nix | 1 +
.../python-packages/drf-pydantic/default.nix | 40 ++
8 files changed, 564 insertions(+), 9 deletions(-)
create mode 100644 deployment/options.nix
create mode 100644 panel/jsonschema.nix
create mode 100644 panel/nix/python-packages/drf-pydantic/default.nix
diff --git a/deployment/default.nix b/deployment/default.nix
index 032cb3e6..4c1ae423 100644
--- a/deployment/default.nix
+++ b/deployment/default.nix
@@ -29,22 +29,23 @@
## From the hosting provider's perspective, the function is meant to be
## partially applied only until here.
-## Information on the specific deployment that we request. This is the
-## information coming from the FediPanel.
-##
-## FIXME: lock step the interface with the definitions in the FediPanel
-panelConfig:
+## Information on the specific deployment that we request.
+## This is the information coming from the FediPanel.
+config:
let
inherit (lib) mkMerge mkIf;
-
+ # TODO(@fricklerhandwerk): misusing this will produce obscure errors that will be truncated by NixOps4
+ panelConfig = (lib.evalModules { modules = [ ./options.nix ]; }).config;
in
## Regular arguments of a NixOps4 deployment module.
{ providers, ... }:
{
- providers = { inherit (nixops4.modules.nixops4Provider) local; };
+ providers = {
+ inherit (nixops4.modules.nixops4Provider) local;
+ };
resources =
let
diff --git a/deployment/options.nix b/deployment/options.nix
new file mode 100644
index 00000000..1de9265f
--- /dev/null
+++ b/deployment/options.nix
@@ -0,0 +1,54 @@
+/**
+ Deployment options as to be presented in the front end.
+
+ These are converted to JSON schema in order to generate front-end forms etc.
+ For this to work, options must not have types `functionTo` or `package` (or `submodule` until [submodule introspection](https://github.com/NixOS/nixpkgs/pull/391544) is merged), and must not access `config` for their default values.
+*/
+{
+ lib,
+ ...
+}:
+let
+ inherit (lib) types mkOption;
+in
+{
+ options = {
+ domain = mkOption {
+ type =
+ with types;
+ enum [
+ "fediversity.net"
+ ];
+ description = ''
+ Apex domain under which the services will be deployed.
+ '';
+ };
+ pixelfed = {
+ enable = lib.mkEnableOption "Pixelfed";
+ };
+ peertube = {
+ enable = lib.mkEnableOption "Peertube";
+ };
+ mastodon = {
+ enable = lib.mkEnableOption "Mastodon";
+ };
+ initialUser = {
+ displayName = mkOption {
+ type = types.str;
+ description = "Display name of the user";
+ };
+ username = mkOption {
+ type = types.str;
+ description = "Username for login";
+ };
+ email = mkOption {
+ type = types.str;
+ description = "User's email address";
+ };
+ password = mkOption {
+ type = types.str;
+ description = "Password for login";
+ };
+ };
+ };
+}
diff --git a/npins/sources.json b/npins/sources.json
index 45efd6b4..530e9aca 100644
--- a/npins/sources.json
+++ b/npins/sources.json
@@ -1,5 +1,16 @@
{
"pins": {
+ "clan-core": {
+ "type": "Git",
+ "repository": {
+ "type": "Git",
+ "url": "https://git.clan.lol/clan/clan-core"
+ },
+ "branch": "main",
+ "revision": "dcb2231332bb4ebc91663914a3bb05ffb875b6d9",
+ "url": null,
+ "hash": "15zvwbzkbm0m2zar3vf698sqk7s84vvprjkfl9hy43jk911qsdgh"
+ },
"htmx": {
"type": "GitRelease",
"repository": {
@@ -30,8 +41,8 @@
"nixpkgs": {
"type": "Channel",
"name": "nixpkgs-unstable",
- "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre711046.8edf06bea5bc/nixexprs.tar.xz",
- "hash": "1mwsn0rvfm603svrq3pca4c51zlix5gkyr4gl6pxhhq3q6xs5s8y"
+ "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre776128.eb0e0f21f15c/nixexprs.tar.xz",
+ "hash": "0l04lkdi3slwwlgwyr8x0argzxcxm16a4hkijfxbjhlj44y1bkif"
}
},
"version": 3
diff --git a/panel/.gitignore b/panel/.gitignore
index 5ab365e7..50b110e3 100644
--- a/panel/.gitignore
+++ b/panel/.gitignore
@@ -1,6 +1,7 @@
# Nix
.direnv
result*
+src/panel/configuration/schema.py
# Python
*.pyc
diff --git a/panel/default.nix b/panel/default.nix
index 767802be..74d813dc 100644
--- a/panel/default.nix
+++ b/panel/default.nix
@@ -12,12 +12,27 @@ let
manage = pkgs.writeScriptBin "manage" ''
exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
'';
+ jsonschema = pkgs.callPackage ./jsonschema.nix { } {
+ includeDefaults = false;
+ };
+ frontend-options = jsonschema.parseModule ../deployment/options.nix;
+ schema = with builtins; toFile "schema.json" (toJSON frontend-options);
+ codegen = "${pkgs.python3Packages.datamodel-code-generator}/bin/datamodel-codegen";
+ pydantic =
+ pkgs.runCommand "schema.py"
+ {
+ }
+ ''
+ ${codegen} --input ${schema} | sed 's/from pydantic/from drf_pydantic/' > $out
+ '';
in
{
+ inherit frontend-options;
shell = pkgs.mkShellNoCC {
inputsFrom = [ (pkgs.callPackage ./nix/package.nix { }) ];
packages = [
pkgs.npins
+ pkgs.jq
manage
];
env = import ./env.nix { inherit lib pkgs; } // {
@@ -26,6 +41,8 @@ in
DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3";
};
shellHook = ''
+ cp -f ${pydantic} ${builtins.toString ./src/panel/configuration/schema.py}
+
ln -sf ${sources.htmx}/dist/htmx.js src/panel/static/htmx.min.js
# in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
# use this directory for testing with local secrets
diff --git a/panel/jsonschema.nix b/panel/jsonschema.nix
new file mode 100644
index 00000000..ca833827
--- /dev/null
+++ b/panel/jsonschema.nix
@@ -0,0 +1,430 @@
+# TODO(@fricklerhandwerk):
+# temporarily copied from
+# https://git.clan.lol/clan/clan-core/src/branch/main/lib/jsonschema/default.nix
+# to work around a bug in JSON Schema generation; this needs a PR to upstream
+{
+ lib,
+}:
+{
+ excludedTypes ? [
+ "functionTo"
+ "package"
+ ],
+ includeDefaults ? true,
+ header ? {
+ "$schema" = "http://json-schema.org/draft-07/schema#";
+ },
+ specialArgs ? { },
+}:
+let
+ # remove _module attribute from options
+ clean = opts: builtins.removeAttrs opts [ "_module" ];
+
+ # throw error if option type is not supported
+ notSupported =
+ option:
+ lib.trace option throw ''
+ option type '${option.type.name}' ('${option.type.description}') not supported by jsonschema converter
+ location: ${lib.concatStringsSep "." option.loc}
+ '';
+
+ # Exclude the option if its type is in the excludedTypes list
+ # or if the option has a defaultText attribute
+ isExcludedOption = option: (lib.elem (option.type.name or null) excludedTypes);
+
+ filterExcluded = lib.filter (opt: !isExcludedOption opt);
+
+ excludedOptionNames = [ "_freeformOptions" ];
+ filterExcludedAttrs = lib.filterAttrs (
+ name: opt: !isExcludedOption opt && !builtins.elem name excludedOptionNames
+ );
+
+ # Filter out options where the visible attribute is set to false
+ filterInvisibleOpts = lib.filterAttrs (_name: opt: opt.visible or true);
+
+ # Constant: Used for the 'any' type
+ allBasicTypes = [
+ "boolean"
+ "integer"
+ "number"
+ "string"
+ "array"
+ "object"
+ "null"
+ ];
+in
+rec {
+ # parses a nixos module to a jsonschema
+ parseModule =
+ module:
+ let
+ evaled = lib.evalModules {
+ modules = [ module ];
+ inherit specialArgs;
+ };
+ in
+ parseOptions evaled.options { };
+
+ # get default value from option
+
+ # Returns '{ default = Value; }'
+ # - '{}' if no default is present.
+ # - Value is "<thunk>" (string literal) if the option has a defaultText attribute. This means we cannot evaluate default safely
+ getDefaultFrom =
+ opt:
+ if !includeDefaults then
+ { }
+ else if opt ? defaultText then
+ {
+ # dont add default to jsonschema. It seems to alter the type
+ # default = "<thunk>";
+ }
+ else
+ lib.optionalAttrs (opt ? default) {
+ default = opt.default;
+ };
+
+ parseSubOptions =
+ {
+ option,
+ prefix ? [ ],
+ }:
+ let
+ subOptions = option.type.getSubOptions option.loc;
+ in
+ parseOptions subOptions {
+ addHeader = false;
+ path = option.loc ++ prefix;
+ };
+
+ makeModuleInfo =
+ {
+ path,
+ defaultText ? null,
+ }:
+ {
+ "$exportedModuleInfo" =
+ {
+ inherit path;
+ }
+ // lib.optionalAttrs (defaultText != null) {
+ inherit defaultText;
+ };
+ };
+
+ # parses a set of evaluated nixos options to a jsonschema
+ parseOptions =
+ options:
+ {
+ # The top-level header object should specify at least the schema version
+ # Can be customized if needed
+ # By default the header is not added to the schema
+ addHeader ? true,
+ path ? [ ],
+ }:
+ let
+ options' = filterInvisibleOpts (filterExcludedAttrs (clean options));
+ # parse options to jsonschema properties
+ properties = lib.mapAttrs (_name: option: (parseOption' (path ++ [ _name ]) option)) options';
+ # TODO: figure out how to handle if prop.anyOf is used
+ isRequired = prop: !(prop ? default || prop.type or null == "object");
+ requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties;
+ required = lib.optionalAttrs (requiredProps != { }) { required = lib.attrNames requiredProps; };
+ header' = if addHeader then header else { };
+
+ # freeformType is a special type
+ freeformDefs = options._module.freeformType.definitions or [ ];
+ checkFreeformDefs =
+ defs:
+ if (builtins.length defs) != 1 then
+ throw "parseOptions: freeformType definitions not supported"
+ else
+ defs;
+ # It seems that freeformType has [ null ]
+ freeformProperties =
+ if freeformDefs != [ ] && builtins.head freeformDefs != null then
+ # freeformType has only one definition
+ parseOption {
+ # options._module.freeformType.definitions
+ type = builtins.head (checkFreeformDefs freeformDefs);
+ _type = "option";
+ loc = path;
+ }
+ else
+ { };
+
+ # Metadata about the module that is made available to the schema via '$propagatedModuleInfo'
+ exportedModuleInfo = makeModuleInfo {
+ inherit path;
+ };
+ in
+ # return jsonschema
+ header'
+ // exportedModuleInfo
+ // required
+ // {
+ type = "object";
+ inherit properties;
+ additionalProperties = false;
+ }
+ // freeformProperties;
+
+ # parses and evaluated nixos option to a jsonschema property definition
+ parseOption = parseOption' [ ];
+ parseOption' =
+ currentPath: option:
+ let
+ default = getDefaultFrom option;
+ example = lib.optionalAttrs (option ? example) {
+ examples =
+ if (builtins.typeOf option.example) == "list" then option.example else [ option.example ];
+ };
+ description = lib.optionalAttrs (option ? description) {
+ description = option.description.text or option.description;
+ };
+ exposedModuleInfo = lib.optionalAttrs true (makeModuleInfo {
+ path = option.loc;
+ defaultText = option.defaultText or null;
+ });
+ in
+ # either type
+ # TODO: if all nested options are excluded, the parent should be excluded too
+ if
+ option.type.name or null == "either" || option.type.name or null == "coercedTo"
+ # return jsonschema property definition for either
+ then
+ let
+ optionsList' = [
+ {
+ type = option.type.nestedTypes.left or option.type.nestedTypes.coercedType;
+ _type = "option";
+ loc = option.loc;
+ }
+ {
+ type = option.type.nestedTypes.right or option.type.nestedTypes.finalType;
+ _type = "option";
+ loc = option.loc;
+ }
+ ];
+ optionsList = filterExcluded optionsList';
+ in
+ exposedModuleInfo // default // example // description // { oneOf = map parseOption optionsList; }
+ # handle nested options (not a submodule)
+ # foo.bar = mkOption { type = str; };
+ else if !option ? _type then
+ (parseOptions option {
+ addHeader = false;
+ path = currentPath;
+ })
+ # throw if not an option
+ else if option._type != "option" && option._type != "option-type" then
+ throw "parseOption: not an option"
+ # parse nullOr
+ else if
+ option.type.name == "nullOr"
+ # return jsonschema property definition for nullOr
+ then
+ let
+ nestedOption = {
+ type = option.type.nestedTypes.elemType;
+ _type = "option";
+ loc = option.loc;
+ };
+ in
+ default
+ // exposedModuleInfo
+ // example
+ // description
+ // {
+ oneOf = [
+ { type = "null"; }
+ ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption));
+ }
+ # parse bool
+ else if
+ option.type.name == "bool"
+ # return jsonschema property definition for bool
+ then
+ exposedModuleInfo // default // example // description // { type = "boolean"; }
+ # parse float
+ else if
+ option.type.name == "float"
+ # return jsonschema property definition for float
+ then
+ exposedModuleInfo // default // example // description // { type = "number"; }
+ # parse int
+ else if
+ (option.type.name == "int" || option.type.name == "positiveInt")
+ # return jsonschema property definition for int
+ then
+ exposedModuleInfo // default // example // description // { type = "integer"; }
+ # TODO: Add support for intMatching in jsonschema
+ # parse port type aka. "unsignedInt16"
+ else if
+ option.type.name == "unsignedInt16"
+ || option.type.name == "unsignedInt"
+ || option.type.name == "pkcs11"
+ || option.type.name == "intBetween"
+ then
+ exposedModuleInfo // default // example // description // { type = "integer"; }
+ # parse string
+ # TODO: parse more precise string types
+ else if
+ option.type.name == "str"
+ || option.type.name == "singleLineStr"
+ || option.type.name == "passwdEntry str"
+ || option.type.name == "passwdEntry path"
+ # return jsonschema property definition for string
+ then
+ exposedModuleInfo // default // example // description // { type = "string"; }
+ # TODO: Add support for stringMatching in jsonschema
+ # parse stringMatching
+ else if lib.strings.hasPrefix "strMatching" option.type.name then
+ exposedModuleInfo // default // example // description // { type = "string"; }
+ # TODO: Add support for separatedString in jsonschema
+ else if lib.strings.hasPrefix "separatedString" option.type.name then
+ exposedModuleInfo // default // example // description // { type = "string"; }
+ # parse string
+ else if
+ option.type.name == "path"
+ # return jsonschema property definition for path
+ then
+ exposedModuleInfo // default // example // description // { type = "string"; }
+ # parse anything
+ else if
+ option.type.name == "anything"
+ # return jsonschema property definition for anything
+ then
+ exposedModuleInfo // default // example // description // { type = allBasicTypes; }
+ # parse unspecified
+ else if
+ option.type.name == "unspecified"
+ # return jsonschema property definition for unspecified
+ then
+ exposedModuleInfo // default // example // description // { type = allBasicTypes; }
+ # parse raw
+ else if
+ option.type.name == "raw"
+ # return jsonschema property definition for raw
+ then
+ exposedModuleInfo // default // example // description // { type = allBasicTypes; }
+ # parse enum
+ else if
+ option.type.name == "enum"
+ # return jsonschema property definition for enum
+ then
+ exposedModuleInfo
+ // default
+ // example
+ // description
+ // {
+ enum = option.type.functor.payload.values;
+ }
+ # parse listOf submodule
+ else if
+ option.type.name == "listOf" && option.type.nestedTypes.elemType.name == "submodule"
+ # return jsonschema property definition for listOf submodule
+ then
+ default
+ // exposedModuleInfo
+ // example
+ // description
+ // {
+ type = "array";
+ items = parseSubOptions { inherit option; };
+ }
+ # parse list
+ else if
+ (option.type.name == "listOf")
+ # return jsonschema property definition for list
+ then
+ let
+ nestedOption = {
+ type = option.type.nestedTypes.elemType;
+ _type = "option";
+ loc = option.loc;
+ };
+ in
+ default
+ // exposedModuleInfo
+ // example
+ // description
+ // {
+ type = "array";
+ }
+ // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; })
+ # parse list of unspecified
+ else if
+ (option.type.name == "listOf") && (option.type.nestedTypes.elemType.name == "unspecified")
+ # return jsonschema property definition for list
+ then
+ exposedModuleInfo // default // example // description // { type = "array"; }
+ # parse attrsOf submodule
+ else if
+ option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule"
+ # return jsonschema property definition for attrsOf submodule
+ then
+ default
+ // exposedModuleInfo
+ // example
+ // description
+ // {
+ type = "object";
+ additionalProperties = parseSubOptions {
+ inherit option;
+ prefix = [ "<name>" ];
+ };
+ }
+ # parse attrs
+ else if
+ option.type.name == "attrs"
+ # return jsonschema property definition for attrs
+ then
+ default
+ // exposedModuleInfo
+ // example
+ // description
+ // {
+ type = "object";
+ additionalProperties = true;
+ }
+ # parse attrsOf
+ # TODO: if nested option is excluded, the parent should be excluded too
+ else if
+ option.type.name == "attrsOf" || option.type.name == "lazyAttrsOf"
+ # return jsonschema property definition for attrs
+ then
+ let
+ nestedOption = {
+ type = option.type.nestedTypes.elemType;
+ _type = "option";
+ loc = option.loc;
+ };
+ in
+ default
+ // exposedModuleInfo
+ // example
+ // description
+ // {
+ type = "object";
+ additionalProperties =
+ if !isExcludedOption nestedOption then
+ parseOption {
+ type = option.type.nestedTypes.elemType;
+ _type = "option";
+ loc = option.loc;
+ }
+ else
+ false;
+ }
+ # parse submodule
+ else if
+ option.type.name == "submodule"
+ # return jsonschema property definition for submodule
+ # then (lib.attrNames (option.type.getSubOptions option.loc).opt)
+ then
+ exposedModuleInfo // example // description // parseSubOptions { inherit option; }
+ # throw error if option type is not supported
+ else
+ notSupported option;
+}
diff --git a/panel/nix/package.nix b/panel/nix/package.nix
index 9337887c..9741b1a9 100644
--- a/panel/nix/package.nix
+++ b/panel/nix/package.nix
@@ -45,6 +45,7 @@ python3.pkgs.buildPythonPackage {
django-debug-toolbar
django-libsass
django-pydantic-field
+ drf-pydantic
django_4
setuptools
];
diff --git a/panel/nix/python-packages/drf-pydantic/default.nix b/panel/nix/python-packages/drf-pydantic/default.nix
new file mode 100644
index 00000000..2fb017a6
--- /dev/null
+++ b/panel/nix/python-packages/drf-pydantic/default.nix
@@ -0,0 +1,40 @@
+{
+ lib,
+ buildPythonPackage,
+ fetchFromGitHub,
+ setuptools,
+ django,
+ pydantic,
+ hatchling,
+ djangorestframework,
+}:
+
+buildPythonPackage rec {
+ pname = "drf-pydantic";
+ version = "v2.7.1";
+ pyproject = true;
+
+ src = fetchFromGitHub {
+ owner = "georgebv";
+ repo = pname;
+ rev = version;
+ hash = "sha256-ABtSoxj/+HHq4hj4Yb6bEiyOl00TCO/9tvBzhv6afxM=";
+ };
+
+ nativeBuildInputs = [
+ setuptools
+ hatchling
+ ];
+
+ propagatedBuildInputs = [
+ django
+ pydantic
+ djangorestframework
+ ];
+
+ meta = with lib; {
+ description = "";
+ homepage = "https://github.com/${src.owner}/${pname}";
+ license = licenses.mit;
+ };
+}