Rebase onto main

Reviewed-on: Fediversity/Fediversity#269
Reviewed-by: Valentin Gagarin <valentin.gagarin@tweag.io>
Co-authored-by: Kiara Grouwstra <kiara@procolix.eu>
Co-committed-by: Kiara Grouwstra <kiara@procolix.eu>

.forgejo/workflows/ci.yaml

@@ -27,3 +27,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.peertube -L
+
+  check-panel:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: cd panel && nix-build -A tests

npins/sources.json

@@ -1,5 +1,20 @@
 {
   "pins": {
+    "htmx": {
+      "type": "GitRelease",
+      "repository": {
+        "type": "GitHub",
+        "owner": "bigskysoftware",
+        "repo": "htmx"
+      },
+      "pre_releases": false,
+      "version_upper_bound": null,
+      "release_prefix": null,
+      "version": "v2.0.4",
+      "revision": "b82cf843e47e575dd8c2ad8fee547d8e2c3bb87f",
+      "url": "https://api.github.com/repos/bigskysoftware/htmx/tarball/v2.0.4",
+      "hash": "1c4zm3b7ym01ijydiss4amd14mv5fbgp1n71vqjk4alc35jlnqy2"
+    },
     "nix-unit": {
       "type": "Git",
       "repository": {

panel/.gitignore

@@ -10,4 +10,5 @@ __pycache__
 db.sqlite3
 src/db.sqlite3
 src/static
+src/panel/static/htmx*
 .credentials

panel/default.nix

@@ -6,28 +6,30 @@
     config = { };
     overlays = [ (import ./nix/overlay.nix) ];
   },
-}:
+}@args:
 let
+  inherit (pkgs) lib;
   manage = pkgs.writeScriptBin "manage" ''
     exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
   '';
 in
-{
+# re-export inputs so they can be overridden granularly
+# (they can't be accessed from the outside any other way)
+args
+// {
   shell = pkgs.mkShellNoCC {
     inputsFrom = [ (pkgs.callPackage ./nix/package.nix { }) ];
     packages = [
       pkgs.npins
       manage
     ];
-    env = {
+    env = import ./env.nix { inherit lib pkgs; } // {
       NPINS_DIRECTORY = toString ../npins;
-      # explicitly use nix, as e.g. lix does not have configurable-impure-env
-      NIX_DIR = pkgs.nix;
-      REPO_DIR = toString ../.;
-      CREDENTIALS_DIRECTORY = builtins.toString ./.credentials;
+      CREDENTIALS_DIRECTORY = toString ./.credentials;
       DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3";
     };
     shellHook = ''
+      ln -sf ${sources.htmx}/dist/htmx.js src/panel/static/htmx.min.js
       # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
       # use this directory for testing with local secrets
       mkdir -p $CREDENTIALS_DIRECTORY
@@ -37,12 +39,7 @@ in
 
   module = import ./nix/configuration.nix;
   tests = pkgs.callPackage ./nix/tests.nix { };
-
-  # re-export inputs so they can be overridden granularly
-  # (they can't be accessed from the outside any other way)
-  inherit
-    sources
-    system
-    pkgs
-    ;
 }
+# re-export inputs so they can be overridden granularly
+# (they can't be accessed from the outside any other way)
+// args

panel/env.nix

@@ -0,0 +1,18 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  inherit (builtins) toString;
+in
+{
+  REPO_DIR = toString ../.;
+  # explicitly use nix, as e.g. lix does not have configurable-impure-env
+  BIN_PATH = lib.makeBinPath [
+    # explicitly use nix, as e.g. lix does not have configurable-impure-env
+    pkgs.nix
+    # nixops error maybe due to our flake git hook: executing 'git': No such file or directory
+    pkgs.git
+  ];
+}

panel/nix/configuration.nix

@@ -23,7 +23,13 @@ let
   cfg = config.services.${name};
   package = pkgs.callPackage ./package.nix { };
 
-  database-url = "sqlite:////var/lib/${name}/db.sqlite3";
+  environment = import ../env.nix { inherit lib pkgs; } // {
+    DATABASE_URL = "sqlite:////var/lib/${name}/db.sqlite3";
+    USER_SETTINGS_FILE = pkgs.concatText "configuration.py" [
+      ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
+      (builtins.toFile "extra-settings.py" cfg.extra-settings)
+    ];
+  };
 
   python-environment = pkgs.python3.withPackages (
     ps: with ps; [
@@ -32,11 +38,6 @@ let
     ]
   );
 
-  configFile = pkgs.concatText "configuration.py" [
-    ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
-    (builtins.toFile "extra-settings.py" cfg.extra-settings)
-  ];
-
   manage-service = writeShellApplication {
     name = "manage";
     text = ''exec ${package}/bin/manage.py "$@"'';
@@ -56,8 +57,9 @@ let
           --property "User=${name}" \
           --property "Group=${name}" \
           --property "WorkingDirectory=/var/lib/${name}" \
-          --property "Environment=DATABASE_URL=${database-url} USER_SETTINGS_FILE=${configFile}" \
-      ''
+          --property "Environment=''
+      + (toString (lib.mapAttrsToList (name: value: "${name}=${value}") environment))
+      + "\" \\\n"
       + optionalString (credentials != [ ]) (
         (concatStringsSep " \\\n" (map (cred: "--property 'LoadCredential=${cred}'") credentials)) + " \\\n"
       )
@@ -190,10 +192,25 @@ in
         RuntimeDirectory = name;
         LogsDirectory = name;
       } // lib.optionalAttrs (credentials != [ ]) { LoadCredential = credentials; };
-      environment = {
-        USER_SETTINGS_FILE = "${configFile}";
-        DATABASE_URL = database-url;
-      };
+
+      # TODO(@fricklerhandwerk):
+      #     Unify handling of runtime settings.
+      #     Right now we have four(!) places where we need to set environment variables, each in its own format:
+      #     - Django's `settings.py` declaring the setting
+      #     - the development environment
+      #     - the `manage` command
+      #     - here, the service configuration
+      #     Ideally we'd set them in two places (development environment and service configuration) but in the same format.
+      #
+      #     For that we need to take into account
+      #     - the different types of settings
+      #       - secrets, which must not end up in the store
+      #       - other values, which can be world-readable
+      #     - ergonomics
+      #       - manipulation should be straightforward in both places; e.g. dumping secrets to a directory that is not git-tracked and adding values to an attrset otherwise
+      #       - error detection and correction; it should be clear where and why one messed up so it can be fixed immediately
+      #     We may also want to test the development environment in CI in order to make sure that we don't break it inadvertently, because misconfiguration due to multiplpe sources of truth wastes a lot of time.
+      inherit environment;
     };
 
     networking.firewall.allowedTCPPorts = [

panel/nix/package.nix

@@ -2,6 +2,7 @@
   lib,
   sqlite,
   python3,
+  sources ? import ../../npins,
 }:
 let
   src =
@@ -58,5 +59,6 @@ python3.pkgs.buildPythonPackage {
     cp -v ${src}/manage.py $out/bin/manage.py
     chmod +x $out/bin/manage.py
     wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
+    cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/
   '';
 }

panel/src/manage.py

@@ -1,4 +1,4 @@
-#!/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/bin/python
+#!/usr/bin/env python
 """Django's command-line utility for administrative tasks."""
 import os
 import sys

panel/src/panel/configuration/forms.py

@@ -39,11 +39,10 @@ class Configuration(BaseModel):
     # XXX: hard-code available apex domains for now,
     #      they will be prefixed by the user name
     class Domain(Enum):
-        EU = "fediversity.eu"
         NET = "fediversity.net"
 
     domain: Domain = Field(
-        default=Domain.EU,
+        default=Domain.NET,
         description="DNS domain where to expose services"
     )
 

panel/src/panel/settings.py

@@ -192,8 +192,8 @@ if user_settings_file is not None:
 #     The correct thing to do here would be using a helper function such as with `get_secret()` that will catch the exception and explain what's wrong and where to put the right values.
 #     Replacing the `USER_SETTINGS_FILE` mechanism following the comment there would probably be a good thing.
 
-# a dir of nix supporting experimental feature `configurable-impure-env`.
-nix_bin_dir=f"{env['NIX_DIR']}/bin/"
+# PATH to expose to launch button
+bin_path=env['BIN_PATH']
 # path of the root flake to trigger nixops from, see #94.
 # to deploy this should be specified, for dev just use a relative path.
 repo_dir = env["REPO_DIR"]

panel/src/panel/static/htmx.min.js

@@ -0,0 +1 @@
+/nix/store/mwqqk0qmldzvv4xj9kq2lbah2flhc44z-source/dist/htmx.js

panel/src/panel/static/style.sass

@@ -3,3 +3,24 @@ body
   margin: 0
   font-family: sans-serif
   box-sizing: border-box
+
+.loader
+  width: 48px
+  height: 48px
+  border: 5px solid #000
+  border-bottom-color: #F34508
+  border-radius: 50%
+  box-sizing: border-box
+  animation: rotation 1s linear infinite
+  display: inline-block
+
+@keyframes rotation
+  0% { transform: rotate(0deg) }
+  100% { transform: rotate(360deg) }
+
+#spinner-container
+  position: absolute
+  top: 50%
+  left: 50%
+  transform: translate(-50%, -50%)
+  display: block

panel/src/panel/templates/base.html

@@ -5,6 +5,7 @@
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
     <meta http-equiv="X-UA-Compatible" content="IE=edge" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <script src="/static/htmx.min.js"></script>
 
     {% load compress %}
     {% compress css %}

panel/src/panel/templates/configuration_form.html

@@ -4,8 +4,20 @@
   {% csrf_token %}
 
   {{ form.as_p }}
+  <button id="deploy-button" class="button"
+          hx-post="{% url 'deployment_status' %}"
+          hx-trigger="click"
+          hx-indicator="#spinner-container"
+          hx-disabled-elt="this"
+          hx-swap="none"
+          name="deploy">
+      Deploy
+  </button>
 
-  <button class="button" type="submit" name="deploy">Deploy</button>
   <button class="button" type="submit" name="save">Save</button>
+
+  <div id="spinner-container" class="htmx-indicator">
+    <span class="loader"></span>
+  </div>
 </form>
 {% endblock %}

panel/src/panel/urls.py

@@ -26,4 +26,5 @@ urlpatterns = [
     path("account/", views.AccountDetail.as_view(), name='account_detail'),
     path("services/", views.ServiceList.as_view(), name='service_list'),
     path("configuration/", views.ConfigurationForm.as_view(), name='configuration_form'),
+    path("deployment/status/", views.DeploymentStatus.as_view(), name='deployment_status'),
 ]

panel/src/panel/views.py

@@ -1,6 +1,7 @@
 from enum import Enum
 import json
 import subprocess
+import os
 
 from django.urls import reverse_lazy
 from django.contrib.auth.mixins import LoginRequiredMixin
@@ -9,9 +10,9 @@ from django.views.generic import TemplateView, DetailView
 from django.views.generic.edit import FormView
 
 from panel import models, settings
+from panel import models
 from panel.configuration import forms
 
-
 class Index(TemplateView):
     template_name = 'index.html'
 
@@ -33,51 +34,12 @@ class ConfigurationForm(LoginRequiredMixin, FormView):
     success_url = reverse_lazy('configuration_form')
     form_class = forms.Form
 
-    def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
-        return context
-
     def get_object(self):
         """Get or create the configuration object for the current user"""
         obj, created = models.Configuration.objects.get_or_create(
             operator=self.request.user,
         )
 
-        # Check for deploy button
-        if "deploy" in self.request.POST.keys():
-            submission = obj.parsed_value.model_dump_json()
-            # FIXME: let the user specify these from the form (#190)
-            dummy_user = {
-              "initialUser": {
-                "displayName": "Testy McTestface",
-                "username": "test",
-                "password": "testtest",
-                "email": "test@test.com",
-              },
-            }
-            # serialize back and forth now we still need to manually inject the dummy user
-            deployment = json.dumps(dummy_user | json.loads(submission))
-            env = {
-                "PATH": settings.nix_bin_dir,
-                # pass in form info to our deployment
-                "DEPLOYMENT": deployment,
-            }
-            cmd = [
-                "nix",
-                "develop",
-                # workaround to pass in info to nixops4 thru env vars, tho impure :(
-                "--extra-experimental-features",
-                "configurable-impure-env",
-                "--command",
-                "nixops4",
-                "apply",
-                "test",
-            ]
-            subprocess.run(
-               cmd,
-               cwd=settings.repo_dir,
-               env=env,
-           )
         return obj
 
     # TODO(@fricklerhandwerk):
@@ -120,9 +82,66 @@ class ConfigurationForm(LoginRequiredMixin, FormView):
         initial.update(self.convert_enums_to_names(config_dict))
         return initial
 
+class Save(ConfigurationForm):
     def form_valid(self, form):
         obj = self.get_object()
         obj.value = form.to_python().model_dump_json()
         obj.save()
 
         return super().form_valid(form)
+
+class DeploymentStatus(ConfigurationForm):
+    def form_valid(self, form):
+        obj = self.get_object()
+        obj.value = form.to_python().model_dump_json()
+        obj.save()
+
+        return super().form_valid(form)
+
+class DeploymentStatus(ConfigurationForm):
+    def form_valid(self, form):
+        obj = self.get_object()
+        obj.value = form.to_python().model_dump_json()
+        obj.save()
+
+        # Check for deploy button
+        if "deploy" in self.request.POST.keys():
+            self.deployment(obj)
+
+        return super().form_valid(form)
+
+    def deployment(self, obj):
+        submission = obj.parsed_value.model_dump_json()
+        # FIXME: let the user specify these from the form (#190)
+        dummy_user = {
+            "initialUser": {
+            "displayName": "Testy McTestface",
+            "username": "test",
+            "password": "testtest",
+            "email": "test@test.com",
+            },
+        }
+        # serialize back and forth now we still need to manually inject the dummy user
+        deployment = json.dumps(dummy_user | json.loads(submission))
+        env = {
+            "PATH": settings.bin_path,
+            # pass in form info to our deployment
+            "DEPLOYMENT": deployment,
+        }
+        cmd = [
+            "nix",
+            "develop",
+            "--extra-experimental-features",
+            "configurable-impure-env",
+            "--command",
+            "nixops4",
+            "apply",
+            "test",
+        ]
+        deployment_result = subprocess.run(
+            cmd,
+            cwd=settings.repo_dir,
+            env=env,
+        )
+        print(deployment_result)
+        return deployment_result