Compare commits

..

7 commits

33 changed files with 271 additions and 1043 deletions

10
.envrc
View file

@ -1,10 +0,0 @@
#!/usr/bin/env bash
# the shebang is ignored, but nice for editors
# shellcheck shell=bash
if type -P lorri &>/dev/null; then
eval "$(lorri direnv --flake .)"
else
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
use flake
fi

View file

@ -7,6 +7,7 @@ on:
push:
branches:
- main
- ci/**
jobs:
check-pre-commit:
@ -27,3 +28,9 @@ jobs:
steps:
- uses: actions/checkout@v4
- run: nix build .#checks.x86_64-linux.peertube -L
check-pixelfed:
runs-on: native
steps:
- uses: actions/checkout@v4
- run: nix build .#checks.x86_64-linux.pixelfed -L

1
.gitignore vendored
View file

@ -6,6 +6,7 @@ tmp/
.proxmox
/.pre-commit-config.yaml
nixos.qcow2
.envrc
.direnv
result*
.nixos-test-history

View file

@ -51,7 +51,6 @@
"keys"
"secrets"
"services"
"panel"
];
files = "^((" + concatStringsSep "|" optin + ")/.*\\.nix|[^/]*\\.nix)$";
in

View file

@ -1,10 +0,0 @@
#!/usr/bin/env bash
# the shebang is ignored, but nice for editors
# shellcheck shell=bash
if type -P lorri &>/dev/null; then
eval "$(lorri direnv)"
else
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
use_nix
fi

13
panel/.gitignore vendored
View file

@ -1,13 +0,0 @@
# Nix
.direnv
result*
# Python
*.pyc
__pycache__
# Django, application-specific
db.sqlite3
src/db.sqlite3
src/static
.credentials

View file

@ -1,46 +0,0 @@
# Fediversity Panel
The Fediversity Panel is a web service for managing Fediversity deployments with a graphical user interface, written in Django.
## Development
- To obtain all tools related to this project, enter the development environment with `nix-shell`.
If you want to do that automatically on entering this directory:
- [Set up `direnv`](https://github.com/nix-community/nix-direnv#installation)
- Run `direnv allow` in the directory where repository is stored on your machine
> **Note**
>
> This is a security boundary, and allows automatically running code from this repository on your machine.
- Run NixOS integration tests and Django unit tests:
```bash
nix-build -A tests
```
- List all available Django management commands with:
```shell-session
manage
```
- Run the server locally
```shell-session
manage runserver
```
- Whenever you add a field in the database schema, run:
```console
manage makemigrations
```
Then before starting the server again, run:
```
manage migrate
```

View file

@ -1,53 +0,0 @@
{
system ? builtins.currentSystem,
sources ? import ../npins,
pkgs ? import sources.nixpkgs {
inherit system;
config = { };
overlays = [ ];
},
}:
let
package =
let
callPackage = pkgs.lib.callPackageWith (pkgs // pkgs.python3.pkgs);
in
callPackage ./nix/package.nix { };
pkgs' = pkgs.extend (_final: _prev: { panel = package; });
manage = pkgs.writeScriptBin "manage" ''
exec ${pkgs.lib.getExe pkgs.python3} ${toString ./src/manage.py} $@
'';
in
{
shell = pkgs.mkShellNoCC {
inputsFrom = [ package ];
packages = [
pkgs.npins
manage
];
env = {
NPINS_DIRECTORY = toString ../npins;
};
shellHook = ''
# in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.
# use this directory for testing with local secrets
mkdir -p .credentials
echo secret > ${builtins.toString ./.credentials}/SECRET_KEY
export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
export DATABASE_URL="sqlite:///${toString ./src}/db.sqlite3"
'';
};
tests = pkgs'.callPackage ./nix/tests.nix { };
inherit package;
# re-export inputs so they can be overridden granularly
# (they can't be accessed from the outside any other way)
inherit
sources
system
pkgs
;
}

View file

@ -1,199 +0,0 @@
{
config,
pkgs,
lib,
...
}:
let
inherit (lib)
concatStringsSep
mapAttrsToList
mkDefault
mkEnableOption
mkIf
mkOption
mkPackageOption
optionalString
types
;
inherit (pkgs) writeShellApplication;
# TODO: configure the name globally for everywhere it's used
name = "panel";
cfg = config.services.${name};
database-url = "sqlite:////var/lib/${name}/db.sqlite3";
python-environment = pkgs.python3.withPackages (
ps: with ps; [
cfg.package
uvicorn
]
);
configFile = pkgs.concatText "configuration.py" [
((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
(builtins.toFile "extra-settings.py" cfg.extra-settings)
];
manage-service = writeShellApplication {
name = "manage";
text = ''exec ${cfg.package}/bin/manage.py "$@"'';
};
manage-admin = writeShellApplication {
# This allows running the `manage` command in the system environment, e.g. to initialise an admin user
# Executing
name = "manage";
text =
''
systemd-run --pty \
--same-dir \
--wait \
--collect \
--service-type=exec \
--unit "manage-${name}.service" \
--property "User=${name}" \
--property "Group=${name}" \
--property "Environment=DATABASE_URL=${database-url} USER_SETTINGS_FILE=${configFile}" \
''
+ optionalString (credentials != [ ]) (
(concatStringsSep " \\\n" (map (cred: "--property 'LoadCredential=${cred}'") credentials)) + " \\\n"
)
+ ''
${lib.getExe manage-service} "$@"
'';
};
credentials = mapAttrsToList (name: secretPath: "${name}:${secretPath}") cfg.secrets;
in
# TODO: for a more clever and generic way of running Django services:
# https://git.dgnum.eu/mdebray/djangonix/
# unlicensed at the time of writing, but surely worth taking some inspiration from...
{
options.services.${name} = {
enable = mkEnableOption "Service configuration for `${name}`";
# NOTE: this requires that the package is present in `pkgs`
package = mkPackageOption pkgs name { };
production = mkOption {
type = types.bool;
default = true;
};
restart = mkOption {
description = "systemd restart behavior";
type = types.enum [
"no"
"on-success"
"on-failure"
"on-abnormal"
"on-abort"
"always"
];
default = "always";
};
domain = mkOption { type = types.str; };
host = mkOption {
type = types.str;
default = "127.0.0.1";
};
port = mkOption {
type = types.port;
default = 8000;
};
settings = mkOption {
type = types.attrsOf types.anything;
default = {
STATIC_ROOT = mkDefault "/var/lib/${name}/static";
DEBUG = mkDefault false;
ALLOWED_HOSTS = mkDefault [
cfg.domain
cfg.host
"localhost"
"[::1]"
];
CSRF_TRUSTED_ORIGINS = mkDefault [ "https://${cfg.domain}" ];
COMPRESS_OFFLINE = true;
LIBSASS_OUTPUT_STYLE = "compressed";
};
description = ''
Django configuration as an attribute set.
Name-value pairs will be converted to Python variable assignments.
'';
};
extra-settings = mkOption {
type = types.lines;
default = "";
description = ''
Django configuration written in Python verbatim.
Contents will be appended to the definitions in `settings`.
'';
};
secrets = mkOption {
type = types.attrsOf types.path;
default = { };
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ manage-admin ];
services = {
nginx.enable = true;
nginx.virtualHosts = {
${cfg.domain} =
{
locations = {
"/".proxyPass = "http://localhost:${toString cfg.port}";
"/static/".alias = "/var/lib/${name}/static/";
};
}
// lib.optionalAttrs cfg.production {
enableACME = true;
forceSSL = true;
};
};
};
users.users.${name} = {
isSystemUser = true;
group = name;
};
users.groups.${name} = { };
systemd.services.${name} = {
description = "${name} ASGI server";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = [
python-environment
manage-service
];
preStart = ''
# Auto-migrate on first run or if the package has changed
versionFile="/var/lib/${name}/package-version"
if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
manage migrate --no-input
manage collectstatic --no-input --clear
manage compress --force
echo ${cfg.package} > "$versionFile"
fi
'';
script = ''
uvicorn ${name}.asgi:application --host ${cfg.host} --port ${toString cfg.port}
'';
serviceConfig = {
Restart = "always";
User = name;
WorkingDirectory = "/var/lib/${name}";
StateDirectory = name;
RuntimeDirectory = name;
LogsDirectory = name;
} // lib.optionalAttrs (credentials != [ ]) { LoadCredential = credentials; };
environment = {
USER_SETTINGS_FILE = "${configFile}";
DATABASE_URL = database-url;
};
};
};
}

View file

@ -1,57 +0,0 @@
{
lib,
buildPythonPackage,
setuptools,
django_4,
django-compressor,
django-libsass,
dj-database-url,
}:
let
src =
with lib.fileset;
toSource {
root = ../src;
fileset = intersection (gitTracked ../../.) ../src;
};
pyproject = with lib; fromTOML pyproject-toml;
# TODO: define this globally
name = "panel";
# TODO: we may want this in a file so it's easier to read statically
version = "0.0.0";
pyproject-toml = ''
[project]
name = "Fediversity-Panel"
version = "${version}"
[tool.setuptools]
packages = [ "${name}" ]
include-package-data = true
'';
in
buildPythonPackage {
pname = name;
inherit (pyproject.project) version;
pyproject = true;
inherit src;
preBuild = ''
echo "recursive-include ${name} *" > MANIFEST.in
cp ${builtins.toFile "source" pyproject-toml} pyproject.toml
'';
propagatedBuildInputs = [
setuptools
django_4
django-compressor
django-libsass
dj-database-url
];
postInstall = ''
mkdir -p $out/bin
cp -v ${src}/manage.py $out/bin/manage.py
chmod +x $out/bin/manage.py
wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
'';
}

View file

@ -1,62 +0,0 @@
{ lib, pkgs }:
let
# TODO: specify project/service name globally
name = "panel";
defaults = {
services.${name} = {
enable = true;
production = false;
restart = "no";
domain = "example.com";
secrets = {
SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret";
};
};
virtualisation = {
memorySize = 2048;
cores = 2;
};
};
in
lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) {
application-tests = {
inherit defaults;
nodes.server = _: { imports = [ ./configuration.nix ]; };
# run all application-level tests managed by Django
# https://docs.djangoproject.com/en/5.0/topics/testing/overview/
testScript = ''
server.succeed("manage test")
'';
};
admin = {
inherit defaults;
nodes.server = _: { imports = [ ./configuration.nix ]; };
# check that the admin interface is served
testScript = ''
server.wait_for_unit("multi-user.target")
server.wait_for_unit("${name}.service")
server.wait_for_open_port(8000)
server.succeed("curl --fail -L -H 'Host: example.org' http://localhost/admin")
'';
};
sass-processing = {
inherit defaults;
nodes.server = _: { imports = [ ./configuration.nix ]; };
extraPythonPackages = ps: with ps; [ beautifulsoup4 ];
skipTypeCheck = true;
# check that stylesheets are pre-processed and served
testScript = ''
from bs4 import BeautifulSoup
server.wait_for_unit("multi-user.target")
server.wait_for_unit("${name}.service")
server.wait_for_open_port(8000)
stdout = server.succeed("curl --fail -H 'Host: example.org' http://localhost")
# the CSS is auto-generated with a hash in the file name
html = BeautifulSoup(stdout, 'html.parser')
css = html.find('link', type="text/css")['href']
server.succeed(f"curl --fail -H 'Host: example.org' http://localhost/{css}")
'';
};
}

View file

@ -1 +0,0 @@
(import ./. { }).shell

View file

@ -1,22 +0,0 @@
#!/nix/store/px2nj16i5gc3d4mnw5l1nclfdxhry61p-python3-3.12.7/bin/python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

View file

@ -1,16 +0,0 @@
"""
ASGI config for panel project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
application = get_asgi_application()

View file

@ -1,171 +0,0 @@
"""
Django settings for panel project.
Generated by 'django-admin startproject' using Django 4.2.16.
For more information on this file, see
https://docs.djangoproject.com/en/4.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
import sys
import os
import importlib.util
import dj_database_url
from os import environ as env
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
def get_secret(name: str, encoding: str = "utf-8") -> str:
credentials_dir = env.get("CREDENTIALS_DIRECTORY")
if credentials_dir is None:
raise RuntimeError("No credentials directory available.")
try:
with open(f"{credentials_dir}/{name}", encoding=encoding) as f:
secret = f.read().removesuffix("\n")
except FileNotFoundError:
raise RuntimeError(f"No secret named {name} found in {credentials_dir}.")
return secret
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = get_secret("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
"panel",
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'compressor',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'panel.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'panel.wsgi.application'
# Database
# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
# https://github.com/jazzband/dj-database-url
DATABASES = {
'default': dj_database_url.config(),
}
# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.2/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.2/howto/static-files/
STATIC_URL = 'static/'
STATIC_ROOT = os.path.join(BASE_DIR, "static/")
STATICFILES_FINDERS = [
"django.contrib.staticfiles.finders.FileSystemFinder",
"django.contrib.staticfiles.finders.AppDirectoriesFinder",
"compressor.finders.CompressorFinder",
]
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
COMPRESS_PRECOMPILERS = [
("text/x-sass", "django_libsass.SassCompiler"),
]
# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# Customization via user settings
# This must be at the end, as it must be able to override the above
# TODO: we may want to do this with a flat environment instead, and get all values from `os.environ.get()`.
# this would make it more obvious which moving parts there are, if that environment is specified for development/staging/production in a visible place.
user_settings_file = env.get("USER_SETTINGS_FILE", None)
if user_settings_file is not None:
spec = importlib.util.spec_from_file_location("user_settings", user_settings_file)
if spec is None or spec.loader is None:
raise RuntimeError("User settings specification failed!")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules["user_settings"] = module
from user_settings import * # noqa: F403 # pyright: ignore [reportMissingImports]

View file

@ -1,5 +0,0 @@
body
padding: 0
margin: 0
font-family: sans-serif
box-sizing: border-box

View file

@ -1,36 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>{% block title %}Fediversity Panel{% endblock %}</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
{% load compress %}
{% compress css %}
<link rel="stylesheet" type="text/x-sass" href="/static/style.sass" />
{% endcompress %}
{% block extra_head %}{% endblock extra_head %}
</head>
<body>
<header>
{% if user.is_authenticated %}
<p>Welcome, {{ user.username }}!</p>
{% else %}
<p>You are not logged in.</p>
{% endif %}
</header>
{% block navigation %}
<nav>
</nav>
{% endblock navigation %}
{% block layout %}
<article>
{% block content %}{% endblock content %}
</article>
{% endblock layout %}
</body>
</html>

View file

@ -1,7 +0,0 @@
{% extends "base.html" %}
{% block content %}
<h1>Fediversity Panel</h1>
<p>Hello world!</p>
{% endblock %}

View file

@ -1,24 +0,0 @@
"""
URL configuration for panel project.
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/4.2/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from panel import views
urlpatterns = [
path('admin/', admin.site.urls),
path('', views.Index.as_view(), name='index'),
]

View file

@ -1,4 +0,0 @@
from django.views.generic import TemplateView
class Index(TemplateView):
template_name = 'index.html'

View file

@ -1,16 +0,0 @@
"""
WSGI config for panel project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'panel.settings')
application = get_wsgi_application()

View file

@ -13,6 +13,14 @@ in
}:
lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
## Pixelfed as packaged in nixpkgs has a permission issue that prevents Nginx
## from being able to serving the images. We fix it here, but this should be
## upstreamed. See https://github.com/NixOS/nixpkgs/issues/235147
services.pixelfed.package = pkgs.pixelfed.overrideAttrs (old: {
patches = (old.patches or [ ]) ++ [ ./pixelfed-group-permissions.patch ];
});
services.garage = {
ensureBuckets = {
pixelfed = {
@ -61,6 +69,8 @@ lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
};
};
users.users.nginx.extraGroups = [ "pixelfed" ];
services.pixelfed.settings = {
## NOTE: This depends on the targets, eg. universities might want control
## over who has an account. We probably want a universal

View file

@ -8,8 +8,8 @@
{
checks = {
mastodon = import ./tests/mastodon.nix { inherit self pkgs; };
pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit self pkgs; };
peertube = import ./tests/peertube.nix { inherit self pkgs; };
pixelfed = import ./tests/pixelfed.nix { inherit self pkgs; };
};
};
}

View file

@ -8,9 +8,6 @@
let
lib = pkgs.lib;
## FIXME: this binding was not used, but maybe we want a side-effect or something?
# rebuildableTest = import ./rebuildableTest.nix pkgs;
testImage = pkgs.copyPathToStore ./green.png;
testImageColour = "#00FF00";

View file

@ -1,4 +1,6 @@
## This file is a basic test of Peertube functionalities.
##
## NOTE: This test needs Peertube >= 6.3.
{ pkgs, self }:
@ -12,7 +14,11 @@ let
pkgs.writers.writePython3Bin "post-video-in-browser"
{
libraries = with pkgs.python3Packages; [ selenium ];
flakeIgnore = [ "E501" ]; # welcome to the 21st century
flakeIgnore = [
"E302"
"E305"
"E501" # welcome to the 21st century
];
}
''
import sys
@ -24,31 +30,18 @@ let
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
print("Create and configure driver...", file=sys.stderr)
options = Options()
print("########################################", file=sys.stderr)
print("A", file=sys.stderr)
options.add_argument("--headless")
print("########################################", file=sys.stderr)
print("B", file=sys.stderr)
service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")
print("########################################", file=sys.stderr)
print("C", file=sys.stderr)
driver = webdriver.Firefox(options=options, service=service)
print("########################################", file=sys.stderr)
print("D", file=sys.stderr)
driver.set_window_size(4096, 2160)
print("########################################", file=sys.stderr)
print("E", file=sys.stderr)
driver.implicitly_wait(360)
print("########################################", file=sys.stderr)
print("F", file=sys.stderr)
wait = WebDriverWait(driver, timeout=360, poll_frequency=10)
print("########################################", file=sys.stderr)
############################################################
# Login
def load(driver, page):
print(f"Loading page {page}...", file=sys.stderr)
driver.get(page)
@ -84,7 +77,6 @@ let
print(f"Done loading page {page}.", file=sys.stderr)
############################################################
# Upload video and take a screenshot
@ -115,9 +107,9 @@ let
# driver.find_element(By.XPATH, "//*[contains(text(), 'Failed to play video')]")
#
# def detect_image_in_screen(d):
# print("Taking a screenshot...", file=sys.stderr)
# print(" Taking a screenshot...", file=sys.stderr)
# d.save_screenshot("/screenshot.png")
# print("Checking it...", file=sys.stderr)
# print(" Checking it...", file=sys.stderr)
# displayed_colours = subprocess.run(
# [
# "magick",
@ -132,7 +124,10 @@ let
# text=True,
# check=True,
# ).stdout
# return bool(re.match(".*#${testVideoColour}.*", displayed_colours, re.S))
# result = bool(re.search("${testVideoColour}", displayed_colours, re.IGNORECASE))
# if not result:
# print(" Could not find the video in the screenshot.", file=sys.stderr)
# return result
#
# print("Wait until the image shows in screen...", file=sys.stderr)
# wait.until(detect_image_in_screen)
@ -175,11 +170,6 @@ pkgs.nixosTest {
../vm/interactive-vm.nix
];
virtualisation = {
memorySize = lib.mkVMOverride 8192;
cores = 8;
};
environment.systemPackages = with pkgs; [
python3
firefox-unwrapped
@ -192,15 +182,15 @@ pkgs.nixosTest {
expect
];
## FIXME: The CI is very slow, so the default timeout of 120s is not
## good enough. We bump it drastically.
systemd.services.postgresql.serviceConfig.TimeoutSec = lib.mkForce 3600;
environment.variables = {
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.peertube.id;
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.peertube.secret;
PT_INITIAL_ROOT_PASSWORD = "testtest";
};
## FIXME: The CI is very slow, so the default timeout of 120s is not
## good enough. We bump it drastically.
systemd.services.postgresql.serviceConfig.TimeoutSec = lib.mkForce 3600;
};
};
@ -211,7 +201,6 @@ pkgs.nixosTest {
# FIXME: I think this trick to look for a password can be replaced by
# services.peertube.serviceEnvironmentFile.PT_INITIAL_ROOT_PASSWORD=testtest
with subtest("Peertube starts"):
server.wait_for_unit("peertube.service")
root_password = server.succeed("acquire-root-password").rstrip()

View file

@ -1,222 +0,0 @@
{ pkgs, self }:
let
lib = pkgs.lib;
## FIXME: this binding was not used but maybe we want a side effect or something?
# rebuildableTest = import ./rebuildableTest.nix pkgs;
email = "test@test.com";
password = "testtest";
# FIXME: Replace all the By.XPATH by By.CSS_SELECTOR.
seleniumImports = ''
import sys
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
'';
seleniumSetup = ''
print("Create and configure driver...", file=sys.stderr)
options = Options()
# options.add_argument("--headless=new")
service = webdriver.ChromeService(executable_path="${lib.getExe pkgs.chromedriver}") # noqa: E501
driver = webdriver.Chrome(options=options, service=service)
driver.implicitly_wait(30)
driver.set_window_size(1280, 960)
'';
seleniumPixelfedLogin = ''
print("Open login page...", file=sys.stderr)
driver.get("http://pixelfed.localhost/login")
print("Enter email...", file=sys.stderr)
driver.find_element(By.ID, "email").send_keys("${email}")
print("Enter password...", file=sys.stderr)
driver.find_element(By.ID, "password").send_keys("${password}")
# FIXME: This is disgusting. Find instead the input type submit in the form
# with action ending in "/login".
print("Click Login button...", file=sys.stderr)
driver.find_element(By.XPATH, "//button[normalize-space()='Login']").click()
'';
## NOTE: `path` must be a valid python string, either a variable or _quoted_.
seleniumTakeScreenshot = path: ''
print("Take screenshot...", file=sys.stderr)
if not driver.save_screenshot(${path}):
raise Exception("selenium could not save screenshot")
'';
seleniumQuit = ''
print("Quitting...", file=sys.stderr)
driver.quit()
'';
seleniumScriptPostPicture =
pkgs.writers.writePython3Bin "selenium-script-post-picture"
{ libraries = with pkgs.python3Packages; [ selenium ]; }
''
import os
import time
${seleniumImports}
from selenium.webdriver.support.wait import WebDriverWait
${seleniumSetup}
${seleniumPixelfedLogin}
time.sleep(3)
media_path = os.environ['POST_MEDIA']
# Find the new post form, fill it in with our pictureand a caption.
print("Click on Create New Post...", file=sys.stderr)
driver.find_element(By.LINK_TEXT, "Create New Post").click()
print("Add file to input element...", file=sys.stderr)
driver.find_element(By.XPATH, "//input[@type='file']").send_keys(media_path)
print("Add a caption", file=sys.stderr)
driver.find_element(By.CSS_SELECTOR, ".media-body textarea").send_keys(
"Fediversity test of image upload to pixelfed with garage storage."
)
time.sleep(3)
print("Click on Post button...", file=sys.stderr)
driver.find_element(By.LINK_TEXT, "Post").click()
# Wait until the post loads, and in particular its picture, then take a
# screenshot of the whole page.
print("Wait for post and image to be loaded...", file=sys.stderr)
img = driver.find_element(
By.XPATH,
"//div[@class='timeline-status-component-content']//img"
)
WebDriverWait(driver, timeout=10).until(
lambda d: d.execute_script("return arguments[0].complete", img)
)
time.sleep(3)
${seleniumTakeScreenshot "\"/home/selenium/screenshot.png\""}
${seleniumQuit}'';
seleniumScriptGetSrc =
pkgs.writers.writePython3Bin "selenium-script-get-src"
{ libraries = with pkgs.python3Packages; [ selenium ]; }
''
${seleniumImports}
${seleniumSetup}
${seleniumPixelfedLogin}
img = driver.find_element(
By.XPATH,
"//div[@class='timeline-status-component-content']//img"
)
# REVIEW: Need to wait for it to be loaded?
print(img.get_attribute('src'))
${seleniumQuit}'';
in
pkgs.nixosTest {
name = "test-pixelfed-garage";
nodes = {
server =
{ config, ... }:
{
services = {
xserver = {
enable = true;
displayManager.lightdm.enable = true;
desktopManager.lxqt.enable = true;
};
displayManager.autoLogin = {
enable = true;
user = "selenium";
};
};
virtualisation.resolution = {
x = 1680;
y = 1050;
};
virtualisation = {
memorySize = lib.mkVMOverride 8192;
cores = 8;
};
imports = with self.nixosModules; [
fediversity
../vm/garage-vm.nix
../vm/pixelfed-vm.nix
];
# TODO: pair down
environment.systemPackages = with pkgs; [
python3
chromium
chromedriver
xh
seleniumScriptPostPicture
seleniumScriptGetSrc
helix
imagemagick
];
environment.variables = {
POST_MEDIA = ./fediversity.png;
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
## without this we get frivolous errors in the logs
MC_REGION = "garage";
};
# chrome does not like being run as root
users.users.selenium = {
isNormalUser = true;
};
};
};
testScript =
{ nodes, ... }:
''
import re
server.start()
with subtest("Pixelfed starts"):
server.wait_for_unit("phpfpm-pixelfed.service")
with subtest("Account creation"):
server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
# NOTE: This could in theory give a false positive if pixelfed changes it's
# colorscheme to include pure green. (see same problem in mastodon-garage.nix).
# TODO: For instance: post a red image and check that the green pixel IS NOT
# there, then post a green image and check that the green pixel IS there.
with subtest("Image displays"):
server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
server.copy_from_vm("/home/selenium/screenshot.png", "")
displayed_colors = server.succeed("magick /home/selenium/screenshot.png -define histogram:unique-colors=true -format %c histogram:info:")
# check that the green image displayed somewhere
image_check = re.match(".*#FF0500.*", displayed_colors, re.S)
if image_check is None:
raise Exception("cannot detect the uploaded image on pixelfed page.")
with subtest("access garage"):
server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
server.succeed("mc ls garage/pixelfed")
with subtest("access image in garage"):
image = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
image = image.rstrip()
if image == "":
raise Exception("image posted to Pixelfed did not get stored in garage")
server.succeed(f"mc cat {image} >/garage-image.png")
garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.png")
image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA")
if garage_image_hash != image_hash:
raise Exception("image stored in garage did not match image uploaded")
with subtest("Check that image comes from garage"):
src = server.succeed("su - selenium -c 'selenium-script-get-src ${email} ${password}'")
if not src.startswith("${nodes.server.fediversity.internal.garage.web.urlForBucket "pixelfed"}"):
raise Exception("image does not come from garage")
'';
}

191
services/tests/pixelfed.nix Normal file
View file

@ -0,0 +1,191 @@
{ pkgs, self }:
let
lib = pkgs.lib;
email = "test@test.com";
password = "testtest";
testPicture = pkgs.copyPathToStore ./green.png;
testPictureColour = "#00FF00";
seleniumScriptPostPicture =
garageBucketUrl:
pkgs.writers.writePython3Bin "selenium-script-post-picture"
{
libraries = with pkgs.python3Packages; [ selenium ];
flakeIgnore = [
"E302"
"E305"
"E501" # welcome to the 21st century
];
}
''
import re
import subprocess
import sys
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
print("Create and configure driver...", file=sys.stderr)
options = Options()
options.add_argument("--headless")
service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}")
driver = webdriver.Firefox(options=options, service=service)
driver.set_window_size(4096, 2160)
driver.implicitly_wait(360)
wait = WebDriverWait(driver, timeout=1080, poll_frequency=120)
############################################################
# Login
print("Open login page...", file=sys.stderr)
driver.get("http://pixelfed.localhost/login")
print("Enter email...", file=sys.stderr)
driver.find_element(By.ID, "email").send_keys("${email}")
print("Enter password...", file=sys.stderr)
driver.find_element(By.ID, "password").send_keys("${password}")
print("Click Login button...", file=sys.stderr)
driver.find_element(By.XPATH, "//button[normalize-space()='Login']").click()
############################################################
# Post picture
# Find the new post form, fill it in with our picture.
print("Click on Create New Post...", file=sys.stderr)
driver.find_element(By.LINK_TEXT, "Create New Post").click()
print("Add file to input element...", file=sys.stderr)
driver.find_element(By.XPATH, "//input[@type='file']").send_keys("${testPicture}")
print("Click on Post button...", file=sys.stderr)
driver.find_element(By.LINK_TEXT, "Post").click()
# Wait until the post loads, and in particular its picture, then take a
# screenshot of the whole page.
print("Wait for post and picture to be loaded...", file=sys.stderr)
img = driver.find_element(By.XPATH, "//div[@class='timeline-status-component-content']//img")
wait.until(lambda d: d.execute_script("return arguments[0].complete", img))
############################################################
# Check that the picture actually shows
def detect_picture_in_screen(d):
print(" Taking a screenshot...", file=sys.stderr)
d.save_screenshot("/home/selenium/screenshot.png")
print(" Checking it...", file=sys.stderr)
displayed_colours = subprocess.run(
[
"magick",
"/home/selenium/screenshot.png",
"-define",
"histogram:unique-colors=true",
"-format",
"%c",
"histogram:info:",
],
capture_output=True,
text=True,
check=True,
).stdout
result = bool(re.search("${testPictureColour}", displayed_colours, re.IGNORECASE))
if not result:
print(" Could not find the picture in the screenshot.", file=sys.stderr)
return result
print("Wait until the picture shows in screen...", file=sys.stderr)
wait.until(detect_picture_in_screen)
print("Picture detected!", file=sys.stderr)
############################################################
# Check that the picture gets to Garage
def detect_src_in_garage(d):
print(" Reload the timeline...", file=sys.stderr)
driver.get("http://pixelfed.localhost/")
print(" Getting the picture's src and checking it...", file=sys.stderr)
img = driver.find_element(By.XPATH, "//div[@class='timeline-status-component-content']//img")
result = img.get_attribute('src').startswith("${garageBucketUrl}")
if not result:
print(" The picture's src does not point to Garage.", file=sys.stderr)
return result
print("Wait until the picture's src points to Garage...", file=sys.stderr)
wait.until(detect_src_in_garage)
print("Picture's src points to Garage!", file=sys.stderr)
############################################################
print("Done; bye!", file=sys.stderr)
driver.close()
'';
in
pkgs.nixosTest {
name = "pixelfed";
nodes = {
server =
{ config, ... }:
{
imports = with self.nixosModules; [
fediversity
../vm/garage-vm.nix
../vm/pixelfed-vm.nix
../vm/interactive-vm.nix
];
environment.systemPackages = with pkgs; [
python3
firefox-unwrapped
geckodriver
xh
(seleniumScriptPostPicture (config.fediversity.internal.garage.web.urlForBucket "pixelfed"))
helix
imagemagick
];
environment.variables = {
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
## without this we get frivolous errors in the logs
MC_REGION = "garage";
};
# Do not run Selenium scripts as root
users.users.selenium.isNormalUser = true;
};
};
testScript =
{ nodes, ... }:
''
server.start()
with subtest("Pixelfed starts"):
server.wait_for_unit("phpfpm-pixelfed.service")
server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
# NOTE: This could in theory give a false positive if pixelfed changes it's
# colorscheme to include pure green. (see same problem in mastodon-garage.nix).
# TODO: For instance: post a red picture and check that the green pixel IS NOT
# there, then post a green picture and check that the green pixel IS there.
with subtest("Post an picture in the browser"):
server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
server.copy_from_vm("/home/selenium/screenshot.png", "")
with subtest("Find picture in garage"):
server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
picture = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
picture = picture.rstrip()
if picture == "":
raise Exception("Could not find any _thumb.png picture stored in Garage")
server.succeed(f"mc cat {picture} > /picture.png")
garage_hash = server.succeed("identify -quiet -format '%#' /picture.png")
hash = server.succeed("identify -quiet -format '%#' ${testPicture}")
if garage_hash != hash:
raise Exception("The picture stored in Garage does not correspond to the original one.")
'';
}

View file

@ -28,7 +28,6 @@ in
inherit value;
}) (filterAttrs (_: { website, ... }: website) cfg.ensureBuckets);
virtualisation.diskSize = 2048;
virtualisation.forwardPorts = [
{
from = "host";

View file

@ -1,9 +1,19 @@
# customize nixos-rebuild build-vm to be a bit more convenient
{ pkgs, ... }:
{ pkgs, lib, ... }:
let
inherit (lib) mkForce;
in
{
# let us log in
users.mutableUsers = false;
users.users.root.hashedPassword = "";
users = {
mutableUsers = false;
users.root = {
hashedPassword = "";
hashedPasswordFile = mkForce null;
};
};
services.openssh = {
enable = true;
settings = {
@ -13,16 +23,11 @@
};
};
# automatically log in
services.getty.autologinUser = "root";
services.getty.helpLine = ''
Type `C-a c` to access the qemu console
Type `C-a x` to quit
'';
# access to convenient things
environment.systemPackages = with pkgs; [
w3m
python3
w3m
xterm # for `resize`
];
environment.loginShellInit = ''
@ -32,23 +37,27 @@
extra-experimental-features = nix-command flakes
'';
virtualisation.memorySize = 2048;
virtualisation = {
memorySize = 8192;
diskSize = 2048;
cores = 8;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 22222;
guest.port = 22;
}
{
from = "host";
host.port = 8080;
guest.port = 80;
}
{
from = "host";
host.port = 8443;
guest.port = 443;
}
];
forwardPorts = [
{
from = "host";
host.port = 22222;
guest.port = 22;
}
{
from = "host";
host.port = 8080;
guest.port = 80;
}
{
from = "host";
host.port = 8443;
guest.port = 443;
}
];
};
}

View file

@ -1,4 +1,4 @@
{ sources ? import ../npins
{ sources ? import ./npins
, system ? builtins.currentSystem
, pkgs ? import sources.nixpkgs {
inherit system;
@ -65,12 +65,12 @@ rec {
tests = with pkgs; with lib;
let
source = fileset.toSource {
root = ../.;
root = ./.;
fileset = fileset.unions [
./default.nix
./tests.nix
./lib.nix
../npins
./npins
];
};
in
@ -86,7 +86,7 @@ rec {
# adding it verbatim will result in <hash'>-<hash>-source, so rename it first
cp -r ${sources.nixpkgs} source
nix-store --add --store "$HOME" source
${getExe nix-unit} --gc-roots-dir "$HOME" --store "$HOME" ${source}/website/tests.nix "$@"
${getExe nix-unit} --gc-roots-dir "$HOME" --store "$HOME" ${source}/tests.nix "$@"
touch $out
'';
}