Even longer waits

.forgejo/workflows/ci.yaml

@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - main
+      - ci/**
 
 jobs:
   check-pre-commit:
@@ -27,3 +28,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: nix build .#checks.x86_64-linux.peertube -L
+
+  check-pixelfed:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.pixelfed -L

services/fediversity/pixelfed.nix

@@ -13,6 +13,14 @@ in
 }:
 
 lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
+
+  ## Pixelfed as packaged in nixpkgs has a permission issue that prevents Nginx
+  ## from being able to serving the images. We fix it here, but this should be
+  ## upstreamed. See https://github.com/NixOS/nixpkgs/issues/235147
+  services.pixelfed.package = pkgs.pixelfed.overrideAttrs (old: {
+    patches = (old.patches or [ ]) ++ [ ./pixelfed-group-permissions.patch ];
+  });
+
   services.garage = {
     ensureBuckets = {
       pixelfed = {
@@ -61,6 +69,8 @@ lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
     };
   };
 
+  users.users.nginx.extraGroups = [ "pixelfed" ];
+
   services.pixelfed.settings = {
     ## NOTE: This depends on the targets, eg. universities might want control
     ## over who has an account. We probably want a universal

services/flake-part.nix

@@ -8,8 +8,8 @@
     {
       checks = {
         mastodon = import ./tests/mastodon.nix { inherit self pkgs; };
-        pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit self pkgs; };
         peertube = import ./tests/peertube.nix { inherit self pkgs; };
+        pixelfed = import ./tests/pixelfed.nix { inherit self pkgs; };
       };
     };
 }

services/tests/mastodon.nix

@@ -8,9 +8,6 @@
 let
   lib = pkgs.lib;
 
-  ## FIXME: this binding was not used, but maybe we want a side-effect or something?
-  # rebuildableTest = import ./rebuildableTest.nix pkgs;
-
   testImage = pkgs.copyPathToStore ./green.png;
   testImageColour = "#00FF00";
 

services/tests/peertube.nix

@@ -1,4 +1,6 @@
 ## This file is a basic test of Peertube functionalities.
+##
+## NOTE: This test needs Peertube >= 6.3.
 
 { pkgs, self }:
 
@@ -12,7 +14,11 @@ let
     pkgs.writers.writePython3Bin "post-video-in-browser"
       {
         libraries = with pkgs.python3Packages; [ selenium ];
-        flakeIgnore = [ "E501" ]; # welcome to the 21st century
+        flakeIgnore = [
+          "E302"
+          "E305"
+          "E501" # welcome to the 21st century
+        ];
       }
       ''
         import sys
@@ -24,31 +30,18 @@ let
         from selenium.webdriver.support.ui import WebDriverWait
         from selenium.common.exceptions import NoSuchElementException
 
+        print("Create and configure driver...", file=sys.stderr)
         options = Options()
-        print("########################################", file=sys.stderr)
-        print("A", file=sys.stderr)
         options.add_argument("--headless")
-        print("########################################", file=sys.stderr)
-        print("B", file=sys.stderr)
         service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")
-        print("########################################", file=sys.stderr)
-        print("C", file=sys.stderr)
         driver = webdriver.Firefox(options=options, service=service)
-        print("########################################", file=sys.stderr)
-        print("D", file=sys.stderr)
         driver.set_window_size(4096, 2160)
-        print("########################################", file=sys.stderr)
-        print("E", file=sys.stderr)
         driver.implicitly_wait(360)
-        print("########################################", file=sys.stderr)
-        print("F", file=sys.stderr)
         wait = WebDriverWait(driver, timeout=360, poll_frequency=10)
-        print("########################################", file=sys.stderr)
 
         ############################################################
         # Login
 
-
         def load(driver, page):
             print(f"Loading page {page}...", file=sys.stderr)
             driver.get(page)
@@ -84,7 +77,6 @@ let
 
             print(f"Done loading page {page}.", file=sys.stderr)
 
-
         ############################################################
         # Upload video and take a screenshot
 
@@ -115,9 +107,9 @@ let
         # driver.find_element(By.XPATH, "//*[contains(text(), 'Failed to play video')]")
         #
         # def detect_image_in_screen(d):
-        #     print("Taking a screenshot...", file=sys.stderr)
+        #     print("  Taking a screenshot...", file=sys.stderr)
         #     d.save_screenshot("/screenshot.png")
-        #     print("Checking it...", file=sys.stderr)
+        #     print("  Checking it...", file=sys.stderr)
         #     displayed_colours = subprocess.run(
         #         [
         #             "magick",
@@ -132,7 +124,10 @@ let
         #         text=True,
         #         check=True,
         #     ).stdout
-        #     return bool(re.match(".*#${testVideoColour}.*", displayed_colours, re.S))
+        #     result = bool(re.search("${testVideoColour}", displayed_colours, re.IGNORECASE))
+        #     if not result:
+        #         print("  Could not find the video in the screenshot.", file=sys.stderr)
+        #     return result
         #
         # print("Wait until the image shows in screen...", file=sys.stderr)
         # wait.until(detect_image_in_screen)
@@ -175,11 +170,6 @@ pkgs.nixosTest {
           ../vm/interactive-vm.nix
         ];
 
-        virtualisation = {
-          memorySize = lib.mkVMOverride 8192;
-          cores = 8;
-        };
-
         environment.systemPackages = with pkgs; [
           python3
           firefox-unwrapped
@@ -192,15 +182,15 @@ pkgs.nixosTest {
           expect
         ];
 
-        ## FIXME: The CI is very slow, so the default timeout of 120s is not
-        ## good enough. We bump it drastically.
-        systemd.services.postgresql.serviceConfig.TimeoutSec = lib.mkForce 3600;
-
         environment.variables = {
           AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.peertube.id;
           AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.peertube.secret;
           PT_INITIAL_ROOT_PASSWORD = "testtest";
         };
+
+        ## FIXME: The CI is very slow, so the default timeout of 120s is not
+        ## good enough. We bump it drastically.
+        systemd.services.postgresql.serviceConfig.TimeoutSec = lib.mkForce 3600;
       };
   };
 
@@ -211,7 +201,6 @@ pkgs.nixosTest {
 
       # FIXME: I think this trick to look for a password can be replaced by
       # services.peertube.serviceEnvironmentFile.PT_INITIAL_ROOT_PASSWORD=testtest
-
       with subtest("Peertube starts"):
         server.wait_for_unit("peertube.service")
         root_password = server.succeed("acquire-root-password").rstrip()

services/tests/pixelfed-garage.nix

@@ -1,222 +0,0 @@
-{ pkgs, self }:
-let
-  lib = pkgs.lib;
-
-  ## FIXME: this binding was not used but maybe we want a side effect or something?
-  # rebuildableTest = import ./rebuildableTest.nix pkgs;
-
-  email = "test@test.com";
-  password = "testtest";
-
-  # FIXME: Replace all the By.XPATH by By.CSS_SELECTOR.
-
-  seleniumImports = ''
-    import sys
-    from selenium import webdriver
-    from selenium.webdriver.common.by import By
-    from selenium.webdriver.chrome.options import Options
-  '';
-
-  seleniumSetup = ''
-    print("Create and configure driver...", file=sys.stderr)
-    options = Options()
-    # options.add_argument("--headless=new")
-    service = webdriver.ChromeService(executable_path="${lib.getExe pkgs.chromedriver}")  # noqa: E501
-    driver = webdriver.Chrome(options=options, service=service)
-    driver.implicitly_wait(30)
-    driver.set_window_size(1280, 960)
-  '';
-
-  seleniumPixelfedLogin = ''
-    print("Open login page...", file=sys.stderr)
-    driver.get("http://pixelfed.localhost/login")
-    print("Enter email...", file=sys.stderr)
-    driver.find_element(By.ID, "email").send_keys("${email}")
-    print("Enter password...", file=sys.stderr)
-    driver.find_element(By.ID, "password").send_keys("${password}")
-    # FIXME: This is disgusting. Find instead the input type submit in the form
-    # with action ending in "/login".
-    print("Click “Login” button...", file=sys.stderr)
-    driver.find_element(By.XPATH, "//button[normalize-space()='Login']").click()
-  '';
-
-  ## NOTE: `path` must be a valid python string, either a variable or _quoted_.
-  seleniumTakeScreenshot = path: ''
-    print("Take screenshot...", file=sys.stderr)
-    if not driver.save_screenshot(${path}):
-        raise Exception("selenium could not save screenshot")
-  '';
-
-  seleniumQuit = ''
-    print("Quitting...", file=sys.stderr)
-    driver.quit()
-  '';
-
-  seleniumScriptPostPicture =
-    pkgs.writers.writePython3Bin "selenium-script-post-picture"
-      { libraries = with pkgs.python3Packages; [ selenium ]; }
-      ''
-        import os
-        import time
-        ${seleniumImports}
-        from selenium.webdriver.support.wait import WebDriverWait
-
-        ${seleniumSetup}
-        ${seleniumPixelfedLogin}
-        time.sleep(3)
-
-        media_path = os.environ['POST_MEDIA']
-
-        # Find the new post form, fill it in with our pictureand a caption.
-        print("Click on “Create New Post”...", file=sys.stderr)
-        driver.find_element(By.LINK_TEXT, "Create New Post").click()
-        print("Add file to input element...", file=sys.stderr)
-        driver.find_element(By.XPATH, "//input[@type='file']").send_keys(media_path)
-        print("Add a caption", file=sys.stderr)
-        driver.find_element(By.CSS_SELECTOR, ".media-body textarea").send_keys(
-            "Fediversity test of image upload to pixelfed with garage storage."
-        )
-        time.sleep(3)
-        print("Click on “Post” button...", file=sys.stderr)
-        driver.find_element(By.LINK_TEXT, "Post").click()
-
-        # Wait until the post loads, and in particular its picture, then take a
-        # screenshot of the whole page.
-        print("Wait for post and image to be loaded...", file=sys.stderr)
-        img = driver.find_element(
-            By.XPATH,
-            "//div[@class='timeline-status-component-content']//img"
-        )
-        WebDriverWait(driver, timeout=10).until(
-            lambda d: d.execute_script("return arguments[0].complete", img)
-        )
-        time.sleep(3)
-
-        ${seleniumTakeScreenshot "\"/home/selenium/screenshot.png\""}
-        ${seleniumQuit}'';
-
-  seleniumScriptGetSrc =
-    pkgs.writers.writePython3Bin "selenium-script-get-src"
-      { libraries = with pkgs.python3Packages; [ selenium ]; }
-      ''
-        ${seleniumImports}
-        ${seleniumSetup}
-        ${seleniumPixelfedLogin}
-
-        img = driver.find_element(
-            By.XPATH,
-            "//div[@class='timeline-status-component-content']//img"
-        )
-        # REVIEW: Need to wait for it to be loaded?
-        print(img.get_attribute('src'))
-
-        ${seleniumQuit}'';
-
-in
-pkgs.nixosTest {
-  name = "test-pixelfed-garage";
-
-  nodes = {
-    server =
-      { config, ... }:
-      {
-
-        services = {
-          xserver = {
-            enable = true;
-            displayManager.lightdm.enable = true;
-            desktopManager.lxqt.enable = true;
-          };
-
-          displayManager.autoLogin = {
-            enable = true;
-            user = "selenium";
-          };
-        };
-        virtualisation.resolution = {
-          x = 1680;
-          y = 1050;
-        };
-
-        virtualisation = {
-          memorySize = lib.mkVMOverride 8192;
-          cores = 8;
-        };
-        imports = with self.nixosModules; [
-          fediversity
-          ../vm/garage-vm.nix
-          ../vm/pixelfed-vm.nix
-        ];
-        # TODO: pair down
-        environment.systemPackages = with pkgs; [
-          python3
-          chromium
-          chromedriver
-          xh
-          seleniumScriptPostPicture
-          seleniumScriptGetSrc
-          helix
-          imagemagick
-        ];
-        environment.variables = {
-          POST_MEDIA = ./fediversity.png;
-          AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
-          AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
-          ## without this we get frivolous errors in the logs
-          MC_REGION = "garage";
-        };
-        # chrome does not like being run as root
-        users.users.selenium = {
-          isNormalUser = true;
-        };
-      };
-  };
-
-  testScript =
-    { nodes, ... }:
-    ''
-      import re
-
-      server.start()
-
-      with subtest("Pixelfed starts"):
-        server.wait_for_unit("phpfpm-pixelfed.service")
-
-      with subtest("Account creation"):
-        server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
-
-      # NOTE: This could in theory give a false positive if pixelfed changes it's
-      # colorscheme to include pure green. (see same problem in mastodon-garage.nix).
-      # TODO: For instance: post a red image and check that the green pixel IS NOT
-      # there, then post a green image and check that the green pixel IS there.
-
-      with subtest("Image displays"):
-        server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
-        server.copy_from_vm("/home/selenium/screenshot.png", "")
-        displayed_colors = server.succeed("magick /home/selenium/screenshot.png -define histogram:unique-colors=true -format %c histogram:info:")
-        # check that the green image displayed somewhere
-        image_check = re.match(".*#FF0500.*", displayed_colors, re.S)
-        if image_check is None:
-          raise Exception("cannot detect the uploaded image on pixelfed page.")
-
-      with subtest("access garage"):
-        server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
-        server.succeed("mc ls garage/pixelfed")
-
-      with subtest("access image in garage"):
-        image = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
-        image = image.rstrip()
-        if image == "":
-          raise Exception("image posted to Pixelfed did not get stored in garage")
-        server.succeed(f"mc cat {image} >/garage-image.png")
-        garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.png")
-        image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA")
-        if garage_image_hash != image_hash:
-          raise Exception("image stored in garage did not match image uploaded")
-
-      with subtest("Check that image comes from garage"):
-        src = server.succeed("su - selenium -c 'selenium-script-get-src ${email} ${password}'")
-        if not src.startswith("${nodes.server.fediversity.internal.garage.web.urlForBucket "pixelfed"}"):
-          raise Exception("image does not come from garage")
-    '';
-}

services/tests/pixelfed.nix

@@ -0,0 +1,191 @@
+{ pkgs, self }:
+
+let
+  lib = pkgs.lib;
+
+  email = "test@test.com";
+  password = "testtest";
+
+  testPicture = pkgs.copyPathToStore ./green.png;
+  testPictureColour = "#00FF00";
+
+  seleniumScriptPostPicture =
+    garageBucketUrl:
+    pkgs.writers.writePython3Bin "selenium-script-post-picture"
+      {
+        libraries = with pkgs.python3Packages; [ selenium ];
+        flakeIgnore = [
+          "E302"
+          "E305"
+          "E501" # welcome to the 21st century
+        ];
+      }
+      ''
+        import re
+        import subprocess
+        import sys
+        from selenium import webdriver
+        from selenium.webdriver.common.by import By
+        from selenium.webdriver.firefox.options import Options
+        from selenium.webdriver.support.ui import WebDriverWait
+
+        print("Create and configure driver...", file=sys.stderr)
+        options = Options()
+        options.add_argument("--headless")
+        service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")
+        driver = webdriver.Firefox(options=options, service=service)
+        driver.set_window_size(4096, 2160)
+        driver.implicitly_wait(360)
+        wait = WebDriverWait(driver, timeout=1080, poll_frequency=120)
+
+        ############################################################
+        # Login
+
+        print("Open login page...", file=sys.stderr)
+        driver.get("http://pixelfed.localhost/login")
+        print("Enter email...", file=sys.stderr)
+        driver.find_element(By.ID, "email").send_keys("${email}")
+        print("Enter password...", file=sys.stderr)
+        driver.find_element(By.ID, "password").send_keys("${password}")
+        print("Click “Login” button...", file=sys.stderr)
+        driver.find_element(By.XPATH, "//button[normalize-space()='Login']").click()
+
+        ############################################################
+        # Post picture
+
+        # Find the new post form, fill it in with our picture.
+        print("Click on “Create New Post”...", file=sys.stderr)
+        driver.find_element(By.LINK_TEXT, "Create New Post").click()
+        print("Add file to input element...", file=sys.stderr)
+        driver.find_element(By.XPATH, "//input[@type='file']").send_keys("${testPicture}")
+        print("Click on “Post” button...", file=sys.stderr)
+        driver.find_element(By.LINK_TEXT, "Post").click()
+
+        # Wait until the post loads, and in particular its picture, then take a
+        # screenshot of the whole page.
+        print("Wait for post and picture to be loaded...", file=sys.stderr)
+        img = driver.find_element(By.XPATH, "//div[@class='timeline-status-component-content']//img")
+        wait.until(lambda d: d.execute_script("return arguments[0].complete", img))
+
+        ############################################################
+        # Check that the picture actually shows
+
+        def detect_picture_in_screen(d):
+            print("  Taking a screenshot...", file=sys.stderr)
+            d.save_screenshot("/home/selenium/screenshot.png")
+            print("  Checking it...", file=sys.stderr)
+            displayed_colours = subprocess.run(
+                [
+                    "magick",
+                    "/home/selenium/screenshot.png",
+                    "-define",
+                    "histogram:unique-colors=true",
+                    "-format",
+                    "%c",
+                    "histogram:info:",
+                ],
+                capture_output=True,
+                text=True,
+                check=True,
+            ).stdout
+            result = bool(re.search("${testPictureColour}", displayed_colours, re.IGNORECASE))
+            if not result:
+                print("  Could not find the picture in the screenshot.", file=sys.stderr)
+            return result
+
+        print("Wait until the picture shows in screen...", file=sys.stderr)
+        wait.until(detect_picture_in_screen)
+        print("Picture detected!", file=sys.stderr)
+
+        ############################################################
+        # Check that the picture gets to Garage
+
+        def detect_src_in_garage(d):
+            print("  Reload the timeline...", file=sys.stderr)
+            driver.get("http://pixelfed.localhost/")
+            print("  Getting the picture's src and checking it...", file=sys.stderr)
+            img = driver.find_element(By.XPATH, "//div[@class='timeline-status-component-content']//img")
+            result = img.get_attribute('src').startswith("${garageBucketUrl}")
+            if not result:
+                print("  The picture's src does not point to Garage.", file=sys.stderr)
+            return result
+
+        print("Wait until the picture's src points to Garage...", file=sys.stderr)
+        wait.until(detect_src_in_garage)
+        print("Picture's src points to Garage!", file=sys.stderr)
+
+        ############################################################
+
+        print("Done; bye!", file=sys.stderr)
+        driver.close()
+      '';
+
+in
+
+pkgs.nixosTest {
+  name = "pixelfed";
+
+  nodes = {
+    server =
+      { config, ... }:
+      {
+        imports = with self.nixosModules; [
+          fediversity
+          ../vm/garage-vm.nix
+          ../vm/pixelfed-vm.nix
+          ../vm/interactive-vm.nix
+        ];
+
+        environment.systemPackages = with pkgs; [
+          python3
+          firefox-unwrapped
+          geckodriver
+          xh
+          (seleniumScriptPostPicture (config.fediversity.internal.garage.web.urlForBucket "pixelfed"))
+          helix
+          imagemagick
+        ];
+
+        environment.variables = {
+          AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
+          AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
+          ## without this we get frivolous errors in the logs
+          MC_REGION = "garage";
+        };
+
+        # Do not run Selenium scripts as root
+        users.users.selenium.isNormalUser = true;
+      };
+  };
+
+  testScript =
+    { nodes, ... }:
+    ''
+      server.start()
+
+      with subtest("Pixelfed starts"):
+        server.wait_for_unit("phpfpm-pixelfed.service")
+        server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
+
+      # NOTE: This could in theory give a false positive if pixelfed changes it's
+      # colorscheme to include pure green. (see same problem in mastodon-garage.nix).
+      # TODO: For instance: post a red picture and check that the green pixel IS NOT
+      # there, then post a green picture and check that the green pixel IS there.
+
+      with subtest("Post an picture in the browser"):
+        server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
+        server.copy_from_vm("/home/selenium/screenshot.png", "")
+
+      with subtest("Find picture in garage"):
+        server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
+        picture = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
+        picture = picture.rstrip()
+        if picture == "":
+          raise Exception("Could not find any _thumb.png picture stored in Garage")
+        server.succeed(f"mc cat {picture} > /picture.png")
+        garage_hash = server.succeed("identify -quiet -format '%#' /picture.png")
+        hash = server.succeed("identify -quiet -format '%#' ${testPicture}")
+        if garage_hash != hash:
+          raise Exception("The picture stored in Garage does not correspond to the original one.")
+    '';
+}

services/vm/garage-vm.nix

@@ -28,7 +28,6 @@ in
       inherit value;
     }) (filterAttrs (_: { website, ... }: website) cfg.ensureBuckets);
 
-  virtualisation.diskSize = 2048;
   virtualisation.forwardPorts = [
     {
       from = "host";

services/vm/interactive-vm.nix

@@ -1,9 +1,19 @@
-# customize nixos-rebuild build-vm to be a bit more convenient
-{ pkgs, ... }:
+{ pkgs, lib, ... }:
+
+let
+  inherit (lib) mkForce;
+
+in
+
 {
-  # let us log in
-  users.mutableUsers = false;
-  users.users.root.hashedPassword = "";
+  users = {
+    mutableUsers = false;
+    users.root = {
+      hashedPassword = "";
+      hashedPasswordFile = mkForce null;
+    };
+  };
+
   services.openssh = {
     enable = true;
     settings = {
@@ -13,16 +23,11 @@
     };
   };
 
-  # automatically log in
   services.getty.autologinUser = "root";
-  services.getty.helpLine = ''
-    Type `C-a c` to access the qemu console
-    Type `C-a x` to quit
-  '';
-  # access to convenient things
+
   environment.systemPackages = with pkgs; [
-    w3m
     python3
+    w3m
     xterm # for `resize`
   ];
   environment.loginShellInit = ''
@@ -32,23 +37,27 @@
     extra-experimental-features = nix-command flakes
   '';
 
-  virtualisation.memorySize = 2048;
+  virtualisation = {
+    memorySize = 8192;
+    diskSize = 2048;
+    cores = 8;
 
-  virtualisation.forwardPorts = [
-    {
-      from = "host";
-      host.port = 22222;
-      guest.port = 22;
-    }
-    {
-      from = "host";
-      host.port = 8080;
-      guest.port = 80;
-    }
-    {
-      from = "host";
-      host.port = 8443;
-      guest.port = 443;
-    }
-  ];
+    forwardPorts = [
+      {
+        from = "host";
+        host.port = 22222;
+        guest.port = 22;
+      }
+      {
+        from = "host";
+        host.port = 8080;
+        guest.port = 80;
+      }
+      {
+        from = "host";
+        host.port = 8443;
+        guest.port = 443;
+      }
+    ];
+  };
 }