add report on 24.11 ZHF hackathon

`S3_HOSTNAME` is only usedful for path-style buckets where Mastodon will
use `<S3_HOSTNAME>/<S3_BUCKET>`. However, we use domain-style, and that
is exactly what `S3_ALIAS_HOST` is for

services/README.md

@@ -40,7 +40,7 @@ NOTE: it sometimes takes a while for the services to start up, and in the meanti
     ```
   - Creating other accounts has to be enabled via the admin interface. `Administration > Configuration > Basic > Enable Signup` or just add an account directly from `Administration > Create user`. But functionality can also be tested from the root account.
 
-- Pixelfed: <http://pixelfed.localhost:8000>
+- Pixelfed: through the reverse proxy at <http://pixelfed.localhost:8080>
   - Account creation via the web interface won't work until we figure out email
   - For now, they can be created on the VM command line
     ```bash

services/fediversity/garage.nix

@@ -216,6 +216,10 @@ in
               proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
               # Disable buffering to a temporary file.
               proxy_max_temp_file_size 0;
+
+              ## NOTE: This page suggests many more options for the object storage
+              ## proxy. We should take a look.
+              ## https://docs.joinmastodon.org/admin/optional/object-storage-proxy/
             '';
           };
         };

services/fediversity/mastodon.nix

@@ -46,9 +46,7 @@ lib.mkIf (config.fediversity.enable && config.fediversity.mastodon.enable) {
       AWS_ACCESS_KEY_ID = snakeoil_key.id;
       AWS_SECRET_ACCESS_KEY = snakeoil_key.secret;
       S3_PROTOCOL = "http";
-      S3_HOSTNAME = config.fediversity.internal.garage.web.rootDomain;
-      # by default it tries to use "<S3_HOSTNAME>/<S3_BUCKET>"
-      S3_ALIAS_HOST = "${S3_BUCKET}.${S3_HOSTNAME}";
+      S3_ALIAS_HOST = "${S3_BUCKET}.${config.fediversity.internal.garage.web.rootDomain}";
       # SEE: the last section in https://docs.joinmastodon.org/admin/optional/object-storage/
       # TODO: can we set up ACLs with garage?
       S3_PERMISSION = "";
@@ -78,9 +76,6 @@ lib.mkIf (config.fediversity.enable && config.fediversity.mastodon.enable) {
       fromAddress = "noreply@${config.fediversity.internal.mastodon.domain}";
       createLocally = false;
     };
-
-    # TODO: this is hardware-dependent. let's figure it out when we have hardware
-    # streamingProcesses = 1;
   };
 
   security.acme = {

services/flake-part.nix

@@ -7,7 +7,7 @@
     { pkgs, ... }:
     {
       checks = {
-        mastodon-garage = import ./tests/mastodon-garage.nix { inherit self pkgs; };
+        mastodon = import ./tests/mastodon.nix { inherit self pkgs; };
         pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit self pkgs; };
       };
     };

services/tests/mastodon.nix

@@ -1,10 +1,19 @@
+## This file is a basic test of Mastodon functionalities.
+##
+## NOTE: This test will fail for Mastodon < 4.3 because of
+## https://github.com/mastodon/mastodon/issues/31145
+
 { pkgs, self }:
+
 let
   lib = pkgs.lib;
 
   ## FIXME: this binding was not used, but maybe we want a side-effect or something?
   # rebuildableTest = import ./rebuildableTest.nix pkgs;
 
+  testImage = pkgs.copyPathToStore ./green.png;
+  testImageColour = "#00FF00";
+
   seleniumScript =
     pkgs.writers.writePython3Bin "selenium-script"
       { libraries = with pkgs.python3Packages; [ selenium ]; }
@@ -14,8 +23,6 @@ let
         from selenium.webdriver.firefox.options import Options
         from selenium.webdriver.support.ui import WebDriverWait
 
-        print(1)
-
         options = Options()
         options.add_argument("--headless")
         # devtools don't show up in headless screenshots
@@ -23,7 +30,7 @@ let
         service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")  # noqa: E501
 
         driver = webdriver.Firefox(options=options, service=service)
-        driver.get("http://mastodon.localhost:55001/public/local")
+        driver.get("http://mastodon.localhost/public/local")
 
         # wait until the statuses load
         WebDriverWait(driver, 90).until(
@@ -34,6 +41,7 @@ let
         driver.close()
       '';
 in
+
 pkgs.nixosTest {
   name = "test-mastodon-garage";
 
@@ -46,6 +54,7 @@ pkgs.nixosTest {
           fediversity
           ../vm/garage-vm.nix
           ../vm/mastodon-vm.nix
+          ../vm/interactive-vm.nix
         ];
         # TODO: pair down
         environment.systemPackages = with pkgs; [
@@ -57,9 +66,9 @@ pkgs.nixosTest {
           seleniumScript
           helix
           imagemagick
+          expect
         ];
         environment.variables = {
-          POST_MEDIA = ./green.png;
           AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.mastodon.id;
           AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.mastodon.secret;
         };
@@ -87,64 +96,67 @@ pkgs.nixosTest {
         if password_match is None:
           raise Exception(f"account creation did not generate a password.\n{account_creation_output}")
         password = password_match.group(1)
-
-      with subtest("TTY Login"):
-        server.wait_until_tty_matches("1", "login: ")
-        server.send_chars("root\n");
+        # print(f"Test user (test@test.com)'s password is: {password}")
 
       with subtest("Log in with toot"):
-        # toot doesn't provide a way to just specify our login details as arguments, so we have to pretend we're typing them in at the prompt
-        server.send_chars("toot login_cli --instance http://mastodon.localhost:55001 --email test@test.com\n")
-        server.wait_until_tty_matches("1", "Password: ")
-        server.send_chars(password + "\n")
-        server.wait_until_tty_matches("1", "Successfully logged in.")
+        # toot doesn't provide a way to just specify our login details as
+        # arguments, so we have to pretend we're typing them in at the prompt;
+        # we use 'expect' for this purpose.
+        server.succeed(f"""
+          expect -c '
+            spawn toot login_cli --instance http://mastodon.localhost:55001 --email test@test.com
+            expect "Password: "
+            send "{password}\\n"
+            interact
+          ' >&2
+        """)
 
-      with subtest("post text"):
+      with subtest("Post a text"):
         server.succeed("echo 'hello mastodon' | toot post")
 
-      with subtest("post image"):
-        server.succeed("toot post --media $POST_MEDIA")
+      with subtest("Post an image"):
+        server.succeed("toot post --media ${testImage}")
 
-      with subtest("access garage"):
+      with subtest("Access garage"):
         server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
         server.succeed("mc ls garage/mastodon")
 
-      with subtest("access image in garage"):
+      with subtest("Access image in garage"):
         image = server.succeed("mc find garage --regex original")
         image = image.rstrip()
         if image == "":
           raise Exception("image posted to mastodon did not get stored in garage")
         server.succeed(f"mc cat {image} >/garage-image.webp")
         garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.webp")
-        image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA")
+        image_hash = server.succeed("identify -quiet -format '%#' ${testImage}")
         if garage_image_hash != image_hash:
           raise Exception("image stored in garage did not match image uploaded")
 
-      with subtest("Content security policy allows garage images"):
+      with subtest("Content-Security-Policy allows garage content"):
         headers = server.succeed("xh -h http://mastodon.localhost:55001/public/local")
         csp_match = None
         # I can't figure out re.MULTILINE
         for header in headers.split("\n"):
-          csp_match =  re.match('^Content-Security-Policy: (.*)$', header)
+          csp_match = re.match('^Content-Security-Policy: (.*)$', header)
           if csp_match is not None:
             break
         if csp_match is None:
           raise Exception("mastodon did not send a content security policy header")
         csp = csp_match.group(1)
-        # the img-src content security policy should include the garage server
+        # the connect-src content security policy should include the garage server
         ## TODO: use `nodes.server.fediversity.internal.garage.api.url` same as above, but beware of escaping the regex. Be careful with port 80 though.
         garage_csp = re.match(".*; img-src[^;]*web\.garage\.localhost.*", csp)
         if garage_csp is None:
-          raise Exception("Mastodon's content security policy does not include garage server. image will not be displayed properly on mastodon.")
+          raise Exception("Mastodon's Content-Security-Policy does not include Garage.")
 
-      # this could in theory give a false positive if mastodon changes it's colorscheme to include pure green.
-      with subtest("image displays"):
+      # this could in theory give a false positive if mastodon changes it's colorscheme to include ${testImageColour}.
+      with subtest("Image displays"):
         server.succeed("selenium-script")
         server.copy_from_vm("/mastodon-screenshot.png", "")
         displayed_colors = server.succeed("convert /mastodon-screenshot.png -define histogram:unique-colors=true -format %c histogram:info:")
-        # check that the green image displayed somewhere
-        green_check = re.match(".*#00FF00.*", displayed_colors, re.S)
-        if green_check is None:
+        # check that the image displayed somewhere
+        image_check = re.match(".*${testImageColour}.*", displayed_colors, re.S)
+        if image_check is None:
           raise Exception("cannot detect the uploaded image on mastodon page.")
     '';
 }

services/tests/pixelfed-garage.nix

@@ -186,7 +186,7 @@ pkgs.nixosTest {
         server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
 
       # NOTE: This could in theory give a false positive if pixelfed changes it's
-      # colorscheme to include pure green. (see same problem in pixelfed-garage.nix).
+      # colorscheme to include pure green. (see same problem in mastodon-garage.nix).
       # TODO: For instance: post a red image and check that the green pixel IS NOT
       # there, then post a green image and check that the green pixel IS there.
 

services/vm/interactive-vm.nix

@@ -32,29 +32,8 @@
     extra-experimental-features = nix-command flakes
   '';
 
-  # no graphics. see nixos-shell
-  virtualisation = {
-    graphics = false;
-    qemu.consoles = [
-      "tty0"
-      "hvc0"
-    ];
-    qemu.options = [
-      "-serial null"
-      "-device virtio-serial"
-      "-chardev stdio,mux=on,id=char0,signal=off"
-      "-mon chardev=char0,mode=readline"
-      "-device virtconsole,chardev=char0,nr=0"
-    ];
-  };
+  virtualisation.memorySize = 2048;
 
-  # we can't forward port 80 or 443, so let's run nginx on a different port
-  networking.firewall.allowedTCPPorts = [
-    8443
-    8080
-  ];
-  services.nginx.defaultSSLListenPort = 8443;
-  services.nginx.defaultHTTPListenPort = 8080;
   virtualisation.forwardPorts = [
     {
       from = "host";
@@ -64,12 +43,12 @@
     {
       from = "host";
       host.port = 8080;
-      guest.port = 8080;
+      guest.port = 80;
     }
     {
       from = "host";
       host.port = 8443;
-      guest.port = 8443;
+      guest.port = 443;
     }
   ];
 }

services/vm/mastodon-vm.nix

@@ -33,15 +33,6 @@
           email = "none";
         };
       };
-
-      virtualisation.memorySize = 2048;
-      virtualisation.forwardPorts = [
-        {
-          from = "host";
-          host.port = 44443;
-          guest.port = 443;
-        }
-      ];
     }
 
     #### run mastodon as development environment
@@ -58,7 +49,6 @@
           BIND = "0.0.0.0";
           # for letter_opener (still doesn't work though)
           REMOTE_DEV = "true";
-          LOCAL_DOMAIN = "${config.fediversity.internal.mastodon.domain}:8443";
         };
       };
 

services/vm/pixelfed-vm.nix

@@ -23,13 +23,4 @@ in
       enableACME = mkVMOverride false;
     };
   };
-
-  virtualisation.memorySize = 2048;
-  virtualisation.forwardPorts = [
-    {
-      from = "host";
-      host.port = 8000;
-      guest.port = 80;
-    }
-  ];
 }

website/content/events/2024-11-zurich-zhf.nix

@@ -0,0 +1,23 @@
+{ config, lib, ... }:
+{
+  collections.events.entry = { link, ... }: {
+    title = "NixOS 24.11 ZHF hackathon";
+    name = lib.mkForce "zhf-24-11";
+    description = "NixOS 24.11 ZHF hackathon in Zürich";
+    start-date = "2024-11-23";
+    end-date = "2024-11-24";
+    start-time = "10:00";
+    end-time = "17:00";
+    location = "OST Campus Rapperswil";
+    body = ''
+      The biannual [Zürich NixOS ZHF hackathon](https://zurich.nix.ug/) has become somewhat of an institution for maintaining the tradition of preparing the upcoming NixOS release.
+
+      The main goal of the two-day gathering is to bring down the number of build failures on the [continuous integration system Hydra](https://status.nixos.org/) before the release: Zero Hydra Failures.
+      It also presents a great opportunity to learn Nix, squash bugs together, get to know each other, discuss current events, and make plans for the future.
+
+      This is the greatest event in the series so far, with more than 40 participants from all over Europe, including many high-profile contributors and maintainers in the Nix ecosystem.
+
+      [Fediversity engineers attended](${link config.collections.news.by-name.zhf-24-11}) to present prototypes and exchange ideas with other developers.
+    '';
+  };
+}

website/content/news/24-11-zurich-zhf.nix

@@ -0,0 +1,22 @@
+{ config, lib, ... }:
+{
+  collections.news.entry = { link, ... }: rec {
+    name = lib.mkForce "zhf-24-11";
+    title = "NixOS 24.11 release hackathon and workshop";
+    description = "Fediversity engineers met in Zürich at a NixOS 24.11 ZHF hackathon";
+    date = "2024-11-28";
+    author = "Valentin Gagarin";
+    summary = ''
+      Fediversity engineers met in Zürich at a [NixOS 24.11 ZHF hackathon](${link config.collections.events.by-name.zhf-24-11}) to present prototypes and exchange ideas with the Nix community.
+    '';
+    body = ''
+      ${summary}
+
+      Robert held a lightning talk on the design of [NixOps4](https://github.com/nixops4/nixops4), which is currently in the prototype stage of development.
+      Before that, Nicolas had already shown an internal demonstration that NixOps4 is capable of deploying multiple NixOS services in the Fediversity test environment.
+
+      In the afternoon, Robert, Valentin, and Koen got together with Eli from [Thymis](https://thymis.io) and Johannes from [Clan](https://clan.lol/) to walk each other through the architecture of their respective systems.
+      This was an extraordinarily fruitful encounter that helped us to identify overlaps and potential for future collaboration!
+    '';
+  };
+}

website/lib.nix

@@ -99,20 +99,22 @@ rec {
   relativePath = path1': path2':
     let
       inherit (lib.path) subpath;
-      inherit (lib) lists;
+      inherit (lib) lists length take drop min max;
 
       path1 = subpath.components path1';
-      prefix1 = with lib; take (length path1 - 1) path1;
+      prefix1 = take (length path1 - 1) path1;
       path2 = subpath.components path2';
-      prefix2 = with lib; take (length path2 - 1) path2;
+      prefix2 = take (length path2 - 1) path2;
 
       commonPrefixLength = with lists;
         findFirstIndex (i: i.fst != i.snd)
-          (length prefix1)
+          (min (length prefix1) (length prefix2))
           (zipLists prefix1 prefix2);
 
+      depth = max 0 (length prefix1 - commonPrefixLength);
+
       relativeComponents = with lists;
-        [ "." ] ++ (replicate (length prefix1 - commonPrefixLength) "..") ++ (drop commonPrefixLength path2);
+        [ "." ] ++ (replicate depth "..") ++ (drop commonPrefixLength path2);
     in
     join "/" relativeComponents;
 

website/structure/collections.nix

@@ -52,12 +52,17 @@ in
         };
         entry = mkOption {
           description = "An entry in the collection";
-          type = types.collection (types.submodule ({
+          type = with types; collection (submodule ({
             imports = [ config.type ];
             _module.args.collection = config;
             process-locations = ls: with lib; concatMap (l: map (p: "${p}/${l}") config.prefixes) ls;
           }));
         };
+        by-name = mkOption {
+          description = "Entries accessible by symbolic name";
+          type = with types; attrsOf attrs;
+          default = with lib; listToAttrs (map (e: { name = e.name; value = e; }) config.entry);
+        };
       };
     }));
   };

website/tests.nix

@@ -4,8 +4,18 @@ let
   inherit (import ./. { }) lib;
 in
 {
-  test-relativePath = {
-    expr = with lib; relativePath "bar" "baz";
-    expected = "./baz";
-  };
+  test-relativePath = with lib;
+    let
+      testData = [
+        { from = "bar"; to = "baz"; expected = "./baz"; }
+        { from = "foo/bar"; to = "foo/baz"; expected = "./baz"; }
+        { from = "foo"; to = "bar/baz"; expected = "./bar/baz"; }
+        { from = "foo/bar"; to = "baz"; expected = "./../baz"; }
+        { from = "foo/bar/baz"; to = "foo"; expected = "./../../foo"; }
+      ];
+    in
+    {
+      expr = map (case: relativePath case.from case.to) testData;
+      expected = map (case: case.expected) testData;
+    };
 }