add CI tests for the panel

removing fediversity.eu from form options as its subdomains are running live services

Reviewed-on: Fediversity/Fediversity#257
Reviewed-by: Kevin Muller <kevin@procolix.com>
Co-authored-by: Kiara Grouwstra <kiara@procolix.eu>
Co-committed-by: Kiara Grouwstra <kiara@procolix.eu>

.forgejo/workflows/ci.yaml

@@ -27,3 +27,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.peertube -L
+
+  check-panel:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: cd panel && nix-build -A tests

flake.nix

@@ -58,9 +58,13 @@
             packages = [
               pkgs.nil
               inputs'.agenix.packages.default
-              inputs'.nixops4.packages.default
+              pkgs.openssh
               pkgs.httpie
               pkgs.jq
+              # exposing this env var as a hack to pass info in from form
+              (inputs'.nixops4.packages.default.overrideAttrs {
+                impureEnvVars = [ "DEPLOYMENT" ];
+              })
             ];
             shellHook = config.pre-commit.installationScript;
           };

infra/flake-part.nix

@@ -143,7 +143,17 @@ in
   ## - We add a “test” deployment with all test machines.
   nixops4Deployments = genAttrs machines makeDeployment' // {
     default = makeDeployment machines;
-    test = makeTestDeployment (fromJSON (readFile ./test-machines/configuration.json));
+    test = makeTestDeployment (
+      fromJSON (
+        let
+          env = builtins.getEnv "DEPLOYMENT";
+        in
+        if env != "" then
+          env
+        else
+          builtins.trace "env var DEPLOYMENT not set, falling back to ./test-machines/configuration.json!" (readFile ./test-machines/configuration.json)
+      )
+    );
   };
   flake.nixosConfigurations =
     genAttrs machines (makeConfiguration false)

infra/machines/fedi201/fedipanel.nix

@@ -4,15 +4,10 @@
 }:
 let
   name = "panel";
-  panel = (import ../../../panel/default.nix { }).package;
 in
 {
   imports = [
-    ../../../panel/nix/configuration.nix
-  ];
-
-  environment.systemPackages = [
-    panel
+    (import ../../../panel { }).module
   ];
 
   security.acme = {
@@ -22,18 +17,11 @@ in
 
   services.${name} = {
     enable = true;
-    package = panel;
     production = true;
     domain = "demo.fediversity.eu";
-    host = "0.0.0.0";
     secrets = {
       SECRET_KEY = config.age.secrets.panel-secret-key.path;
     };
     port = 8000;
-    settings = {
-      DATABASE_URL = "sqlite:///var/lib/${name}/db.sqlite3";
-      CREDENTIALS_DIRECTORY = "/var/lib/${name}/.credentials";
-      STATIC_ROOT = "/var/lib/${name}/static";
-    };
   };
 }

panel/default.nix

@@ -8,36 +8,36 @@
   },
 }:
 let
-  package = pkgs.callPackage ./nix/package.nix { };
-
-  pkgs' = pkgs.extend (_final: _prev: { panel = package; });
-
+  inherit (pkgs) lib;
   manage = pkgs.writeScriptBin "manage" ''
     exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
   '';
 in
 {
   shell = pkgs.mkShellNoCC {
-    inputsFrom = [ package ];
+    inputsFrom = [ (pkgs.callPackage ./nix/package.nix { }) ];
     packages = [
       pkgs.npins
       manage
     ];
     env = {
       NPINS_DIRECTORY = toString ../npins;
+      # explicitly use nix, as e.g. lix does not have configurable-impure-env
+      NIX_BIN = lib.getExe pkgs.nix;
+      REPO_DIR = toString ../.;
+      CREDENTIALS_DIRECTORY = builtins.toString ./.credentials;
+      DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3";
     };
     shellHook = ''
       # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
       # use this directory for testing with local secrets
-      mkdir -p .credentials
+      mkdir -p $CREDENTIALS_DIRECTORY
       echo secret > ${builtins.toString ./.credentials}/SECRET_KEY
-      export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
-      export DATABASE_URL="sqlite:///${toString ./src}/db.sqlite3"
     '';
   };
 
-  tests = pkgs'.callPackage ./nix/tests.nix { };
-  inherit package;
+  module = import ./nix/configuration.nix;
+  tests = pkgs.callPackage ./nix/tests.nix { };
 
   # re-export inputs so they can be overridden granularly
   # (they can't be accessed from the outside any other way)

panel/nix/configuration.nix

@@ -12,7 +12,6 @@ let
     mkEnableOption
     mkIf
     mkOption
-    mkPackageOption
     optionalString
     types
     ;
@@ -22,23 +21,15 @@ let
   name = "panel";
 
   cfg = config.services.${name};
+  package = pkgs.callPackage ./package.nix { };
 
   database-url = "sqlite:////var/lib/${name}/db.sqlite3";
 
   python-environment = pkgs.python3.withPackages (
-    ps:
-    with ps;
-    [
+    ps: with ps; [
+      package
       uvicorn
-      cfg.package
-      dj-database-url
-      django-compressor
-      django-debug-toolbar
-      django-libsass
-      django_4
-      setuptools
     ]
-    ++ cfg.package.propagatedBuildInputs
   );
 
   configFile = pkgs.concatText "configuration.py" [
@@ -48,7 +39,7 @@ let
 
   manage-service = writeShellApplication {
     name = "manage";
-    text = ''exec ${cfg.package}/bin/manage.py "$@"'';
+    text = ''exec ${package}/bin/manage.py "$@"'';
   };
 
   manage-admin = writeShellApplication {
@@ -58,13 +49,13 @@ let
     text =
       ''
         systemd-run --pty \
-          --same-dir \
           --wait \
           --collect \
           --service-type=exec \
           --unit "manage-${name}.service" \
           --property "User=${name}" \
           --property "Group=${name}" \
+          --property "WorkingDirectory=/var/lib/${name}" \
           --property "Environment=DATABASE_URL=${database-url} USER_SETTINGS_FILE=${configFile}" \
       ''
       + optionalString (credentials != [ ]) (
@@ -83,8 +74,6 @@ in
 {
   options.services.${name} = {
     enable = mkEnableOption "Service configuration for `${name}`";
-    # NOTE: this requires that the package is present in `pkgs`
-    package = mkPackageOption pkgs name { };
     production = mkOption {
       type = types.bool;
       default = true;
@@ -145,6 +134,8 @@ in
   };
 
   config = mkIf cfg.enable {
+    nixpkgs.overlays = [ (import ./overlay.nix) ];
+
     environment.systemPackages = [ manage-admin ];
 
     services = {
@@ -181,16 +172,15 @@ in
       preStart = ''
         # Auto-migrate on first run or if the package has changed
         versionFile="/var/lib/${name}/package-version"
-        if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
+        if [[ $(cat "$versionFile" 2>/dev/null) != ${package} ]]; then
           manage migrate --no-input
           manage collectstatic --no-input --clear
           manage compress --force
-          echo ${cfg.package} > "$versionFile"
+          echo ${package} > "$versionFile"
         fi
       '';
       script = ''
-        export PYTHONPATH=$PYTHONPATH:${cfg.package}/lib/python3.12/site-packages
-        ${python-environment}/bin/python -m uvicorn ${name}.asgi:application --host ${cfg.host} --port ${toString cfg.port}
+        uvicorn ${name}.asgi:application --host ${cfg.host} --port ${toString cfg.port}
       '';
       serviceConfig = {
         Restart = "always";

panel/nix/tests.nix

@@ -3,6 +3,8 @@ let
   # TODO: specify project/service name globally
   name = "panel";
   defaults = {
+    # XXX: we have to duplicate this here despite it being defined in the service module, otherwise the test framework will error out
+    nixpkgs.overlays = lib.mkForce [ (import ./overlay.nix) ];
     services.${name} = {
       enable = true;
       production = false;
@@ -26,6 +28,7 @@ lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; }))
     # run all application-level tests managed by Django
     # https://docs.djangoproject.com/en/5.0/topics/testing/overview/
     testScript = ''
+      server.wait_for_unit("${name}.service")
       server.succeed("manage test ${name}")
     '';
   };
@@ -34,7 +37,6 @@ lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; }))
     nodes.server = _: { imports = [ ./configuration.nix ]; };
     # check that the admin interface is served
     testScript = ''
-      server.wait_for_unit("multi-user.target")
       server.wait_for_unit("${name}.service")
       server.wait_for_open_port(8000)
       server.succeed("curl --fail -L -H 'Host: example.org' http://localhost/admin")
@@ -45,11 +47,11 @@ lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; }))
     inherit defaults;
     nodes.server = _: { imports = [ ./configuration.nix ]; };
     extraPythonPackages = ps: with ps; [ beautifulsoup4 ];
+    # type checking on `beautifulsoup4` will error out
     skipTypeCheck = true;
     # check that stylesheets are pre-processed and served
     testScript = ''
       from bs4 import BeautifulSoup
-      server.wait_for_unit("multi-user.target")
       server.wait_for_unit("${name}.service")
       server.wait_for_open_port(8000)
       stdout = server.succeed("curl --fail -H 'Host: example.org' http://localhost")

panel/src/panel/configuration/forms.py

@@ -39,11 +39,10 @@ class Configuration(BaseModel):
     # XXX: hard-code available apex domains for now,
     #      they will be prefixed by the user name
     class Domain(Enum):
-        EU = "fediversity.eu"
         NET = "fediversity.net"
 
     domain: Domain = Field(
-        default=Domain.EU,
+        default=Domain.NET,
         description="DNS domain where to expose services"
     )
 

panel/src/panel/settings.py

@@ -26,6 +26,8 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
 
 def get_secret(name: str, encoding: str = "utf-8") -> str:
+    # In the NixOS deployment, this variable is set by `systemd` via `LoadCredential`
+    # https://systemd.io/CREDENTIALS/
     credentials_dir = env.get("CREDENTIALS_DIRECTORY")
 
     if credentials_dir is None:
@@ -171,8 +173,9 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 # Customization via user settings
 # This must be at the end, as it must be able to override the above
-# TODO: we may want to do this with a flat environment instead, and get all values from `os.environ.get()`.
-#       this would make it more obvious which moving parts there are, if that environment is specified for development/staging/production in a visible place.
+# TODO(@fricklerhandwerk):
+#     we may want to do this with a flat environment instead, and get all values from `os.environ.get()`.
+#     this would make it more obvious which moving parts there are, if that environment is specified for development/staging/production in a visible place.
 user_settings_file = env.get("USER_SETTINGS_FILE", None)
 if user_settings_file is not None:
     spec = importlib.util.spec_from_file_location("user_settings", user_settings_file)
@@ -182,3 +185,15 @@ if user_settings_file is not None:
     spec.loader.exec_module(module)
     sys.modules["user_settings"] = module
     from user_settings import *  # noqa: F403 # pyright: ignore [reportMissingImports]
+
+# non-Django application settings
+
+# TODO(@fricklerhandwerk):
+#     The correct thing to do here would be using a helper function such as with `get_secret()` that will catch the exception and explain what's wrong and where to put the right values.
+#     Replacing the `USER_SETTINGS_FILE` mechanism following the comment there would probably be a good thing.
+
+# a dir of nix supporting experimental feature `configurable-impure-env`.
+nix_bin=env['NIX_BIN']
+# path of the root flake to trigger nixops from, see #94.
+# to deploy this should be specified, for dev just use a relative path.
+repo_dir = env["REPO_DIR"]

panel/src/panel/templates/configuration_form.html

@@ -5,7 +5,7 @@
 
   {{ form.as_p }}
 
-  <button class="button" disabled>Deploy</button>
-  <button class="button" type="submit" >Save</button>
+  <button class="button" type="submit" name="deploy">Deploy</button>
+  <button class="button" type="submit" name="save">Save</button>
 </form>
 {% endblock %}

panel/src/panel/views.py

@@ -1,4 +1,6 @@
 from enum import Enum
+import json
+import subprocess
 
 from django.urls import reverse_lazy
 from django.contrib.auth.mixins import LoginRequiredMixin
@@ -6,7 +8,7 @@ from django.contrib.auth.models import User
 from django.views.generic import TemplateView, DetailView
 from django.views.generic.edit import FormView
 
-from panel import models
+from panel import models, settings
 from panel.configuration import forms
 
 
@@ -41,6 +43,40 @@ class ConfigurationForm(LoginRequiredMixin, FormView):
             operator=self.request.user,
         )
 
+        # Check for deploy button
+        if "deploy" in self.request.POST.keys():
+            submission = obj.parsed_value.model_dump_json()
+            # FIXME: let the user specify these from the form (#190)
+            dummy_user = {
+              "initialUser": {
+                "displayName": "Testy McTestface",
+                "username": "test",
+                "password": "testtest",
+                "email": "test@test.com",
+              },
+            }
+            # serialize back and forth now we still need to manually inject the dummy user
+            deployment = json.dumps(dummy_user | json.loads(submission))
+            env = {
+                # pass in form info to our deployment
+                "DEPLOYMENT": deployment,
+            }
+            cmd = [
+                settings.nix_bin,
+                "develop",
+                # workaround to pass in info to nixops4 thru env vars, tho impure :(
+                "--extra-experimental-features",
+                "configurable-impure-env",
+                "--command",
+                "nixops4",
+                "apply",
+                "test",
+            ]
+            subprocess.run(
+               cmd,
+               cwd=settings.repo_dir,
+               env=env,
+           )
         return obj
 
     # TODO(@fricklerhandwerk):