Format everything, RFC-style

This commit is contained in:
Nicolas Jeannerod 2024-11-11 17:25:42 +01:00 committed by Valentin Gagarin
parent 9e234e7b2d
commit 81176a1a9a
16 changed files with 684 additions and 535 deletions

View file

@ -5,7 +5,8 @@ let
inherit (lib) mkOption mkEnableOption mkForce; inherit (lib) mkOption mkEnableOption mkForce;
inherit (lib.types) types; inherit (lib.types) types;
in { in
{
imports = [ imports = [
./garage.nix ./garage.nix
./mastodon.nix ./mastodon.nix
@ -33,7 +34,7 @@ in {
temp = mkOption { temp = mkOption {
description = "options that are only used while developing; should be removed eventually"; description = "options that are only used while developing; should be removed eventually";
default = {}; default = { };
type = types.submodule { type = types.submodule {
options = { options = {
cores = mkOption { cores = mkOption {
@ -51,7 +52,7 @@ in {
internal = mkOption { internal = mkOption {
description = "options that are only meant to be used internally; change at your own risk"; description = "options that are only meant to be used internally; change at your own risk";
default = {}; default = { };
type = types.submodule { type = types.submodule {
options = { options = {
garage = { garage = {

View file

@ -8,27 +8,49 @@ let
in in
# TODO: expand to a multi-machine setup # TODO: expand to a multi-machine setup
{ config, lib, pkgs, ... }: {
config,
lib,
pkgs,
...
}:
let let
inherit (builtins) toString; inherit (builtins) toString;
inherit (lib) types mkOption mkEnableOption optionalString concatStringsSep; inherit (lib)
types
mkOption
mkEnableOption
optionalString
concatStringsSep
;
inherit (lib.strings) escapeShellArg; inherit (lib.strings) escapeShellArg;
inherit (lib.attrsets) filterAttrs mapAttrs'; inherit (lib.attrsets) filterAttrs mapAttrs';
cfg = config.services.garage; cfg = config.services.garage;
fedicfg = config.fediversity.internal.garage; fedicfg = config.fediversity.internal.garage;
concatMapAttrs = scriptFn: attrset: concatStringsSep "\n" (lib.mapAttrsToList scriptFn attrset); concatMapAttrs = scriptFn: attrset: concatStringsSep "\n" (lib.mapAttrsToList scriptFn attrset);
ensureBucketScriptFn = bucket: { website, aliases, corsRules }: ensureBucketScriptFn =
bucket:
{
website,
aliases,
corsRules,
}:
let let
bucketArg = escapeShellArg bucket; bucketArg = escapeShellArg bucket;
corsRulesJSON = escapeShellArg (builtins.toJSON { corsRulesJSON = escapeShellArg (
CORSRules = [{ builtins.toJSON {
AllowedHeaders = corsRules.allowedHeaders; CORSRules = [
AllowedMethods = corsRules.allowedMethods; {
AllowedOrigins = corsRules.allowedOrigins; AllowedHeaders = corsRules.allowedHeaders;
}]; AllowedMethods = corsRules.allowedMethods;
}); AllowedOrigins = corsRules.allowedOrigins;
in '' }
];
}
);
in
''
# garage bucket info tells us if the bucket already exists # garage bucket info tells us if the bucket already exists
garage bucket info ${bucketArg} || garage bucket create ${bucketArg} garage bucket info ${bucketArg} || garage bucket create ${bucketArg}
@ -37,9 +59,11 @@ let
garage bucket website --allow ${bucketArg} garage bucket website --allow ${bucketArg}
''} ''}
${concatStringsSep "\n" (map (alias: '' ${concatStringsSep "\n" (
garage bucket alias ${bucketArg} ${escapeShellArg alias} map (alias: ''
'') aliases)} garage bucket alias ${bucketArg} ${escapeShellArg alias}
'') aliases
)}
${optionalString corsRules.enable '' ${optionalString corsRules.enable ''
garage bucket allow --read --write --owner ${bucketArg} --key tmp garage bucket allow --read --write --owner ${bucketArg} --key tmp
@ -49,15 +73,29 @@ let
''} ''}
''; '';
ensureBucketsScript = concatMapAttrs ensureBucketScriptFn cfg.ensureBuckets; ensureBucketsScript = concatMapAttrs ensureBucketScriptFn cfg.ensureBuckets;
ensureAccessScriptFn = key: bucket: { read, write, owner }: '' ensureAccessScriptFn =
garage bucket allow ${optionalString read "--read"} ${optionalString write "--write"} ${optionalString owner "--owner"} \ key: bucket:
${escapeShellArg bucket} --key ${escapeShellArg key} {
''; read,
ensureKeyScriptFn = key: {id, secret, ensureAccess}: '' write,
## FIXME: Check whether the key exist and skip this step if that is the case. Get rid of this `|| :` owner,
garage key import --yes -n ${escapeShellArg key} ${escapeShellArg id} ${escapeShellArg secret} || : }:
${concatMapAttrs (ensureAccessScriptFn key) ensureAccess} ''
''; garage bucket allow ${optionalString read "--read"} ${optionalString write "--write"} ${optionalString owner "--owner"} \
${escapeShellArg bucket} --key ${escapeShellArg key}
'';
ensureKeyScriptFn =
key:
{
id,
secret,
ensureAccess,
}:
''
## FIXME: Check whether the key exist and skip this step if that is the case. Get rid of this `|| :`
garage key import --yes -n ${escapeShellArg key} ${escapeShellArg id} ${escapeShellArg secret} || :
${concatMapAttrs (ensureAccessScriptFn key) ensureAccess}
'';
ensureKeysScript = concatMapAttrs ensureKeyScriptFn cfg.ensureKeys; ensureKeysScript = concatMapAttrs ensureKeyScriptFn cfg.ensureKeys;
in in
@ -66,76 +104,85 @@ in
options = { options = {
services.garage = { services.garage = {
ensureBuckets = mkOption { ensureBuckets = mkOption {
type = types.attrsOf (types.submodule { type = types.attrsOf (
options = { types.submodule {
website = mkOption { options = {
type = types.bool; website = mkOption {
default = false; type = types.bool;
}; default = false;
# I think setting corsRules should allow another website to show images from your bucket
corsRules = {
enable = mkEnableOption "CORS Rules";
allowedHeaders = mkOption {
type = types.listOf types.str;
default = [];
}; };
allowedMethods = mkOption { # I think setting corsRules should allow another website to show images from your bucket
type = types.listOf types.str; corsRules = {
default = []; enable = mkEnableOption "CORS Rules";
allowedHeaders = mkOption {
type = types.listOf types.str;
default = [ ];
};
allowedMethods = mkOption {
type = types.listOf types.str;
default = [ ];
};
allowedOrigins = mkOption {
type = types.listOf types.str;
default = [ ];
};
}; };
allowedOrigins = mkOption { aliases = mkOption {
type = types.listOf types.str; type = types.listOf types.str;
default = []; default = [ ];
}; };
}; };
aliases = mkOption { }
type = types.listOf types.str; );
default = []; default = { };
};
};
});
default = {};
}; };
ensureKeys = mkOption { ensureKeys = mkOption {
type = types.attrsOf (types.submodule { type = types.attrsOf (
# TODO: these should be managed as secrets, not in the nix store types.submodule {
options = { # TODO: these should be managed as secrets, not in the nix store
id = mkOption { options = {
type = types.str; id = mkOption {
type = types.str;
};
secret = mkOption {
type = types.str;
};
# TODO: assert at least one of these is true
# NOTE: this currently needs to be done at the top level module
ensureAccess = mkOption {
type = types.attrsOf (
types.submodule {
options = {
read = mkOption {
type = types.bool;
default = false;
};
write = mkOption {
type = types.bool;
default = false;
};
owner = mkOption {
type = types.bool;
default = false;
};
};
}
);
default = [ ];
};
}; };
secret = mkOption { }
type = types.str; );
}; default = { };
# TODO: assert at least one of these is true
# NOTE: this currently needs to be done at the top level module
ensureAccess = mkOption {
type = types.attrsOf (types.submodule {
options = {
read = mkOption {
type = types.bool;
default = false;
};
write = mkOption {
type = types.bool;
default = false;
};
owner = mkOption {
type = types.bool;
default = false;
};
};
});
default = [];
};
};
});
default = {};
}; };
}; };
}; };
config = lib.mkIf config.fediversity.enable { config = lib.mkIf config.fediversity.enable {
environment.systemPackages = [ pkgs.minio-client pkgs.awscli ]; environment.systemPackages = [
pkgs.minio-client
pkgs.awscli
];
networking.firewall.allowedTCPPorts = [ networking.firewall.allowedTCPPorts = [
fedicfg.rpc.port fedicfg.rpc.port
@ -178,9 +225,11 @@ in
''; '';
}; };
}; };
in mapAttrs' in
(bucket: _: {name = fedicfg.web.domainForBucket bucket; inherit value;}) mapAttrs' (bucket: _: {
(filterAttrs (_: {website, ...}: website) cfg.ensureBuckets); name = fedicfg.web.domainForBucket bucket;
inherit value;
}) (filterAttrs (_: { website, ... }: website) cfg.ensureBuckets);
systemd.services.ensure-garage = { systemd.services.ensure-garage = {
after = [ "garage.service" ]; after = [ "garage.service" ];
@ -188,7 +237,11 @@ in
serviceConfig = { serviceConfig = {
Type = "oneshot"; Type = "oneshot";
}; };
path = [ cfg.package pkgs.perl pkgs.awscli ]; path = [
cfg.package
pkgs.perl
pkgs.awscli
];
script = '' script = ''
set -xeuo pipefail set -xeuo pipefail

View file

@ -5,10 +5,15 @@ let
}; };
in in
{ config, lib, pkgs, ... }: {
config,
lib,
pkgs,
...
}:
lib.mkIf (config.fediversity.enable && config.fediversity.mastodon.enable) { lib.mkIf (config.fediversity.enable && config.fediversity.mastodon.enable) {
#### garage setup #### garage setup
services.garage = { services.garage = {
ensureBuckets = { ensureBuckets = {
mastodon = { mastodon = {
@ -58,7 +63,10 @@ lib.mkIf (config.fediversity.enable && config.fediversity.mastodon.enable) {
#### mastodon setup #### mastodon setup
# open up access to the mastodon web interface. 80 is necessary if only for ACME # open up access to the mastodon web interface. 80 is necessary if only for ACME
networking.firewall.allowedTCPPorts = [ 80 443 ]; networking.firewall.allowedTCPPorts = [
80
443
];
services.mastodon = { services.mastodon = {
enable = true; enable = true;

View file

@ -5,10 +5,18 @@ let
}; };
in in
{ config, lib, pkgs, ... }: {
config,
lib,
pkgs,
...
}:
lib.mkIf (config.fediversity.enable && config.fediversity.peertube.enable) { lib.mkIf (config.fediversity.enable && config.fediversity.peertube.enable) {
networking.firewall.allowedTCPPorts = [ 80 443 ]; networking.firewall.allowedTCPPorts = [
80
443
];
services.garage = { services.garage = {
ensureBuckets = { ensureBuckets = {
@ -22,7 +30,7 @@ lib.mkIf (config.fediversity.enable && config.fediversity.peertube.enable) {
allowedOrigins = [ "*" ]; allowedOrigins = [ "*" ];
}; };
}; };
# TODO: these are too broad, after getting everything works narrow it down to the domain we actually want # TODO: these are too broad, after getting everything works narrow it down to the domain we actually want
peertube-playlists = { peertube-playlists = {
website = true; website = true;
corsRules = { corsRules = {

View file

@ -5,7 +5,12 @@ let
}; };
in in
{ config, lib, pkgs, ... }: {
config,
lib,
pkgs,
...
}:
lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) { lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
services.garage = { services.garage = {
@ -80,5 +85,8 @@ lib.mkIf (config.fediversity.enable && config.fediversity.pixelfed.enable) {
after = [ "ensure-garage.service" ]; after = [ "ensure-garage.service" ];
}; };
networking.firewall.allowedTCPPorts = [ 80 443 ]; networking.firewall.allowedTCPPorts = [
80
443
];
} }

View file

@ -9,113 +9,123 @@
disko.url = "github:nix-community/disko"; disko.url = "github:nix-community/disko";
}; };
outputs = { self, nixpkgs, nixpkgs-latest, pixelfed, disko }: outputs =
let {
system = "x86_64-linux"; self,
lib = nixpkgs.lib; nixpkgs,
pkgs = nixpkgs.legacyPackages.${system}; nixpkgs-latest,
pkgsLatest = nixpkgs-latest.legacyPackages.${system}; pixelfed,
bleedingFediverseOverlay = (self: super: { disko,
pixelfed = pkgsLatest.pixelfed.overrideAttrs (old: { }:
src = pixelfed; let
patches = (old.patches or [ ]) ++ [ ./fediversity/pixelfed-group-permissions.patch ]; system = "x86_64-linux";
}); lib = nixpkgs.lib;
## TODO: give mastodon, peertube the same treatment pkgs = nixpkgs.legacyPackages.${system};
}); pkgsLatest = nixpkgs-latest.legacyPackages.${system};
in { bleedingFediverseOverlay = (
nixosModules = { self: super: {
## Bleeding-edge fediverse packages pixelfed = pkgsLatest.pixelfed.overrideAttrs (old: {
bleedingFediverse = { src = pixelfed;
nixpkgs.overlays = [ bleedingFediverseOverlay ]; patches = (old.patches or [ ]) ++ [ ./fediversity/pixelfed-group-permissions.patch ];
}; });
## Fediversity modules ## TODO: give mastodon, peertube the same treatment
fediversity = import ./fediversity; }
);
in
{
nixosModules = {
## Bleeding-edge fediverse packages
bleedingFediverse = {
nixpkgs.overlays = [ bleedingFediverseOverlay ];
};
## Fediversity modules
fediversity = import ./fediversity;
## VM-specific modules ## VM-specific modules
interactive-vm = import ./vm/interactive-vm.nix; interactive-vm = import ./vm/interactive-vm.nix;
garage-vm = import ./vm/garage-vm.nix; garage-vm = import ./vm/garage-vm.nix;
mastodon-vm = import ./vm/mastodon-vm.nix; mastodon-vm = import ./vm/mastodon-vm.nix;
peertube-vm = import ./vm/peertube-vm.nix; peertube-vm = import ./vm/peertube-vm.nix;
pixelfed-vm = import ./vm/pixelfed-vm.nix; pixelfed-vm = import ./vm/pixelfed-vm.nix;
disk-layout = import ./disk-layout.nix; disk-layout = import ./disk-layout.nix;
};
nixosConfigurations = {
mastodon = nixpkgs.lib.nixosSystem {
inherit system;
modules = with self.nixosModules; [
disko.nixosModules.default
disk-layout
bleedingFediverse
fediversity
interactive-vm
garage-vm
mastodon-vm
];
}; };
peertube = nixpkgs.lib.nixosSystem { nixosConfigurations = {
inherit system; mastodon = nixpkgs.lib.nixosSystem {
modules = with self.nixosModules; [ inherit system;
disko.nixosModules.default modules = with self.nixosModules; [
disk-layout disko.nixosModules.default
bleedingFediverse disk-layout
fediversity bleedingFediverse
interactive-vm fediversity
garage-vm interactive-vm
peertube-vm garage-vm
]; mastodon-vm
];
};
peertube = nixpkgs.lib.nixosSystem {
inherit system;
modules = with self.nixosModules; [
disko.nixosModules.default
disk-layout
bleedingFediverse
fediversity
interactive-vm
garage-vm
peertube-vm
];
};
pixelfed = nixpkgs.lib.nixosSystem {
inherit system;
modules = with self.nixosModules; [
disko.nixosModules.default
disk-layout
bleedingFediverse
fediversity
interactive-vm
garage-vm
pixelfed-vm
];
};
all = nixpkgs.lib.nixosSystem {
inherit system;
modules = with self.nixosModules; [
disko.nixosModules.default
disk-layout
bleedingFediverse
fediversity
interactive-vm
garage-vm
peertube-vm
pixelfed-vm
mastodon-vm
];
};
}; };
pixelfed = nixpkgs.lib.nixosSystem { ## Fully-feature ISO installer
inherit system; mkInstaller = import ./installer.nix;
modules = with self.nixosModules; [ installers = lib.mapAttrs (_: config: self.mkInstaller nixpkgs config) self.nixosConfigurations;
disko.nixosModules.default
disk-layout deploy =
bleedingFediverse let
fediversity deployCommand = (pkgs.callPackage ./deploy.nix { });
interactive-vm in
garage-vm lib.mapAttrs (name: config: deployCommand name config) self.nixosConfigurations;
pixelfed-vm
]; checks.${system} = {
mastodon-garage = import ./tests/mastodon-garage.nix { inherit pkgs self; };
pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit pkgs self; };
}; };
all = nixpkgs.lib.nixosSystem { devShells.${system}.default = pkgs.mkShell {
inherit system; inputs = with pkgs; [
modules = with self.nixosModules; [ nil
disko.nixosModules.default
disk-layout
bleedingFediverse
fediversity
interactive-vm
garage-vm
peertube-vm
pixelfed-vm
mastodon-vm
]; ];
}; };
}; };
## Fully-feature ISO installer
mkInstaller = import ./installer.nix;
installers = lib.mapAttrs (_: config: self.mkInstaller nixpkgs config) self.nixosConfigurations;
deploy =
let
deployCommand = (pkgs.callPackage ./deploy.nix { });
in
lib.mapAttrs (name: config: deployCommand name config) self.nixosConfigurations;
checks.${system} = {
mastodon-garage = import ./tests/mastodon-garage.nix { inherit pkgs self; };
pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit pkgs self; };
};
devShells.${system}.default = pkgs.mkShell {
inputs = with pkgs; [
nil
];
};
};
} }

View file

@ -4,15 +4,22 @@
WARNING: Running this installer will format the target disk! WARNING: Running this installer will format the target disk!
*/ */
{ nixpkgs, {
hostKeys ? {} nixpkgs,
hostKeys ? { },
}: }:
machine: machine:
let let
inherit (builtins) concatStringsSep attrValues mapAttrs; inherit (builtins) concatStringsSep attrValues mapAttrs;
installer = { config, pkgs, lib, ... }: installer =
{
config,
pkgs,
lib,
...
}:
let let
bootstrap = pkgs.writeShellApplication { bootstrap = pkgs.writeShellApplication {
name = "bootstrap"; name = "bootstrap";
@ -20,39 +27,35 @@ let
text = '' text = ''
${machine.config.system.build.diskoScript} ${machine.config.system.build.diskoScript}
nixos-install --no-root-password --no-channel-copy --system ${machine.config.system.build.toplevel} nixos-install --no-root-password --no-channel-copy --system ${machine.config.system.build.toplevel}
${ ${concatStringsSep "\n" (
concatStringsSep "\n" ( attrValues (
attrValues ( mapAttrs (kind: keys: ''
mapAttrs cp ${keys.private} /mnt/etc/ssh/ssh_host_${kind}_key
(kind: keys: '' chmod 600 /mnt/etc/ssh/ssh_host_${kind}_key
cp ${keys.private} /mnt/etc/ssh/ssh_host_${kind}_key cp ${keys.public} /mnt/etc/ssh/ssh_host_${kind}_key.pub
chmod 600 /mnt/etc/ssh/ssh_host_${kind}_key chmod 644 /mnt/etc/ssh/ssh_host_${kind}_key.pub
cp ${keys.public} /mnt/etc/ssh/ssh_host_${kind}_key.pub '') hostKeys
chmod 644 /mnt/etc/ssh/ssh_host_${kind}_key.pub
'')
hostKeys
)
) )
} )}
poweroff poweroff
''; '';
}; };
in in
{ {
imports = [ imports = [
"${nixpkgs}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix" "${nixpkgs}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix"
]; ];
nixpkgs.hostPlatform = "x86_64-linux"; nixpkgs.hostPlatform = "x86_64-linux";
services.getty.autologinUser = lib.mkForce "root"; services.getty.autologinUser = lib.mkForce "root";
programs.bash.loginShellInit = nixpkgs.lib.getExe bootstrap; programs.bash.loginShellInit = nixpkgs.lib.getExe bootstrap;
isoImage = { isoImage = {
compressImage = false; compressImage = false;
squashfsCompression = "lz4"; squashfsCompression = "lz4";
isoName = lib.mkForce "installer.iso"; isoName = lib.mkForce "installer.iso";
## ^^ FIXME: Use a more interesting name or keep the default name and ## ^^ FIXME: Use a more interesting name or keep the default name and
## use `isoImage.isoName` in the tests. ## use `isoImage.isoName` in the tests.
};
}; };
};
in in
(nixpkgs.lib.nixosSystem { modules = [installer]; }).config.system.build.isoImage (nixpkgs.lib.nixosSystem { modules = [ installer ]; }).config.system.build.isoImage

View file

@ -2,143 +2,149 @@
let let
lib = pkgs.lib; lib = pkgs.lib;
rebuildableTest = import ./rebuildableTest.nix pkgs; rebuildableTest = import ./rebuildableTest.nix pkgs;
seleniumScript = pkgs.writers.writePython3Bin "selenium-script" seleniumScript =
{ pkgs.writers.writePython3Bin "selenium-script"
libraries = with pkgs.python3Packages; [ selenium ]; {
} '' libraries = with pkgs.python3Packages; [ selenium ];
from selenium import webdriver }
from selenium.webdriver.common.by import By ''
from selenium.webdriver.firefox.options import Options from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
print(1) print(1)
options = Options() options = Options()
options.add_argument("--headless") options.add_argument("--headless")
# devtools don't show up in headless screenshots # devtools don't show up in headless screenshots
# options.add_argument("-devtools") # options.add_argument("-devtools")
service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}") # noqa: E501 service = webdriver.FirefoxService(executable_path="${lib.getExe pkgs.geckodriver}") # noqa: E501
driver = webdriver.Firefox(options=options, service=service) driver = webdriver.Firefox(options=options, service=service)
driver.get("http://mastodon.localhost:55001/public/local") driver.get("http://mastodon.localhost:55001/public/local")
# wait until the statuses load # wait until the statuses load
WebDriverWait(driver, 90).until( WebDriverWait(driver, 90).until(
lambda x: x.find_element(By.CLASS_NAME, "status")) lambda x: x.find_element(By.CLASS_NAME, "status"))
driver.save_screenshot("/mastodon-screenshot.png") driver.save_screenshot("/mastodon-screenshot.png")
driver.close() driver.close()
''; '';
in in
pkgs.nixosTest { pkgs.nixosTest {
name = "test-mastodon-garage"; name = "test-mastodon-garage";
nodes = { nodes = {
server = { config, ... }: { server =
virtualisation.memorySize = lib.mkVMOverride 4096; { config, ... }:
imports = with self.nixosModules; [ {
bleedingFediverse virtualisation.memorySize = lib.mkVMOverride 4096;
fediversity imports = with self.nixosModules; [
garage-vm bleedingFediverse
mastodon-vm fediversity
]; garage-vm
# TODO: pair down mastodon-vm
environment.systemPackages = with pkgs; [ ];
python3 # TODO: pair down
firefox-unwrapped environment.systemPackages = with pkgs; [
geckodriver python3
toot firefox-unwrapped
xh geckodriver
seleniumScript toot
helix xh
imagemagick seleniumScript
]; helix
environment.variables = { imagemagick
POST_MEDIA = ./green.png; ];
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.mastodon.id; environment.variables = {
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.mastodon.secret; POST_MEDIA = ./green.png;
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.mastodon.id;
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.mastodon.secret;
};
}; };
};
}; };
testScript = { nodes, ... }: '' testScript =
import re { nodes, ... }:
import time ''
import re
import time
server.start() server.start()
with subtest("Mastodon starts"): with subtest("Mastodon starts"):
server.wait_for_unit("mastodon-web.service") server.wait_for_unit("mastodon-web.service")
# make sure mastodon is fully up and running before we interact with it # make sure mastodon is fully up and running before we interact with it
# TODO: is there a way to test for this? # TODO: is there a way to test for this?
time.sleep(180) time.sleep(180)
with subtest("Account creation"): with subtest("Account creation"):
account_creation_output = server.succeed("mastodon-tootctl accounts create test --email test@test.com --confirmed --approve") account_creation_output = server.succeed("mastodon-tootctl accounts create test --email test@test.com --confirmed --approve")
password_match = re.match('.*New password: ([^\n]*).*', account_creation_output, re.S) password_match = re.match('.*New password: ([^\n]*).*', account_creation_output, re.S)
if password_match is None: if password_match is None:
raise Exception(f"account creation did not generate a password.\n{account_creation_output}") raise Exception(f"account creation did not generate a password.\n{account_creation_output}")
password = password_match.group(1) password = password_match.group(1)
with subtest("TTY Login"): with subtest("TTY Login"):
server.wait_until_tty_matches("1", "login: ") server.wait_until_tty_matches("1", "login: ")
server.send_chars("root\n"); server.send_chars("root\n");
with subtest("Log in with toot"): with subtest("Log in with toot"):
# toot doesn't provide a way to just specify our login details as arguments, so we have to pretend we're typing them in at the prompt # toot doesn't provide a way to just specify our login details as arguments, so we have to pretend we're typing them in at the prompt
server.send_chars("toot login_cli --instance http://mastodon.localhost:55001 --email test@test.com\n") server.send_chars("toot login_cli --instance http://mastodon.localhost:55001 --email test@test.com\n")
server.wait_until_tty_matches("1", "Password: ") server.wait_until_tty_matches("1", "Password: ")
server.send_chars(password + "\n") server.send_chars(password + "\n")
server.wait_until_tty_matches("1", "Successfully logged in.") server.wait_until_tty_matches("1", "Successfully logged in.")
with subtest("post text"): with subtest("post text"):
server.succeed("echo 'hello mastodon' | toot post") server.succeed("echo 'hello mastodon' | toot post")
with subtest("post image"): with subtest("post image"):
server.succeed("toot post --media $POST_MEDIA") server.succeed("toot post --media $POST_MEDIA")
with subtest("access garage"): with subtest("access garage"):
server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY") server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
server.succeed("mc ls garage/mastodon") server.succeed("mc ls garage/mastodon")
with subtest("access image in garage"): with subtest("access image in garage"):
image = server.succeed("mc find garage --regex original") image = server.succeed("mc find garage --regex original")
image = image.rstrip() image = image.rstrip()
if image == "": if image == "":
raise Exception("image posted to mastodon did not get stored in garage") raise Exception("image posted to mastodon did not get stored in garage")
server.succeed(f"mc cat {image} >/garage-image.webp") server.succeed(f"mc cat {image} >/garage-image.webp")
garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.webp") garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.webp")
image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA") image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA")
if garage_image_hash != image_hash: if garage_image_hash != image_hash:
raise Exception("image stored in garage did not match image uploaded") raise Exception("image stored in garage did not match image uploaded")
with subtest("Content security policy allows garage images"): with subtest("Content security policy allows garage images"):
headers = server.succeed("xh -h http://mastodon.localhost:55001/public/local") headers = server.succeed("xh -h http://mastodon.localhost:55001/public/local")
csp_match = None csp_match = None
# I can't figure out re.MULTILINE # I can't figure out re.MULTILINE
for header in headers.split("\n"): for header in headers.split("\n"):
csp_match = re.match('^Content-Security-Policy: (.*)$', header) csp_match = re.match('^Content-Security-Policy: (.*)$', header)
if csp_match is not None: if csp_match is not None:
break break
if csp_match is None: if csp_match is None:
raise Exception("mastodon did not send a content security policy header") raise Exception("mastodon did not send a content security policy header")
csp = csp_match.group(1) csp = csp_match.group(1)
# the img-src content security policy should include the garage server # the img-src content security policy should include the garage server
## TODO: use `nodes.server.fediversity.internal.garage.api.url` same as above, but beware of escaping the regex. Be careful with port 80 though. ## TODO: use `nodes.server.fediversity.internal.garage.api.url` same as above, but beware of escaping the regex. Be careful with port 80 though.
garage_csp = re.match(".*; img-src[^;]*web\.garage\.localhost.*", csp) garage_csp = re.match(".*; img-src[^;]*web\.garage\.localhost.*", csp)
if garage_csp is None: if garage_csp is None:
raise Exception("Mastodon's content security policy does not include garage server. image will not be displayed properly on mastodon.") raise Exception("Mastodon's content security policy does not include garage server. image will not be displayed properly on mastodon.")
# this could in theory give a false positive if mastodon changes it's colorscheme to include pure green. # this could in theory give a false positive if mastodon changes it's colorscheme to include pure green.
with subtest("image displays"): with subtest("image displays"):
server.succeed("selenium-script") server.succeed("selenium-script")
server.copy_from_vm("/mastodon-screenshot.png", "") server.copy_from_vm("/mastodon-screenshot.png", "")
displayed_colors = server.succeed("convert /mastodon-screenshot.png -define histogram:unique-colors=true -format %c histogram:info:") displayed_colors = server.succeed("convert /mastodon-screenshot.png -define histogram:unique-colors=true -format %c histogram:info:")
# check that the green image displayed somewhere # check that the green image displayed somewhere
green_check = re.match(".*#00FF00.*", displayed_colors, re.S) green_check = re.match(".*#00FF00.*", displayed_colors, re.S)
if green_check is None: if green_check is None:
raise Exception("cannot detect the uploaded image on mastodon page.") raise Exception("cannot detect the uploaded image on mastodon page.")
''; '';
} }

View file

@ -50,166 +50,176 @@ let
driver.quit() driver.quit()
''; '';
seleniumScriptPostPicture = pkgs.writers.writePython3Bin "selenium-script-post-picture" seleniumScriptPostPicture =
{ pkgs.writers.writePython3Bin "selenium-script-post-picture"
libraries = with pkgs.python3Packages; [ selenium ]; {
} '' libraries = with pkgs.python3Packages; [ selenium ];
import os }
import time ''
${seleniumImports} import os
from selenium.webdriver.support.wait import WebDriverWait import time
${seleniumImports}
from selenium.webdriver.support.wait import WebDriverWait
${seleniumSetup} ${seleniumSetup}
${seleniumPixelfedLogin} ${seleniumPixelfedLogin}
time.sleep(3) time.sleep(3)
media_path = os.environ['POST_MEDIA'] media_path = os.environ['POST_MEDIA']
# Find the new post form, fill it in with our pictureand a caption. # Find the new post form, fill it in with our pictureand a caption.
print("Click on Create New Post...", file=sys.stderr) print("Click on Create New Post...", file=sys.stderr)
driver.find_element(By.LINK_TEXT, "Create New Post").click() driver.find_element(By.LINK_TEXT, "Create New Post").click()
print("Add file to input element...", file=sys.stderr) print("Add file to input element...", file=sys.stderr)
driver.find_element(By.XPATH, "//input[@type='file']").send_keys(media_path) driver.find_element(By.XPATH, "//input[@type='file']").send_keys(media_path)
print("Add a caption", file=sys.stderr) print("Add a caption", file=sys.stderr)
driver.find_element(By.CSS_SELECTOR, ".media-body textarea").send_keys( driver.find_element(By.CSS_SELECTOR, ".media-body textarea").send_keys(
"Fediversity test of image upload to pixelfed with garage storage." "Fediversity test of image upload to pixelfed with garage storage."
) )
time.sleep(3) time.sleep(3)
print("Click on Post button...", file=sys.stderr) print("Click on Post button...", file=sys.stderr)
driver.find_element(By.LINK_TEXT, "Post").click() driver.find_element(By.LINK_TEXT, "Post").click()
# Wait until the post loads, and in particular its picture, then take a # Wait until the post loads, and in particular its picture, then take a
# screenshot of the whole page. # screenshot of the whole page.
print("Wait for post and image to be loaded...", file=sys.stderr) print("Wait for post and image to be loaded...", file=sys.stderr)
img = driver.find_element( img = driver.find_element(
By.XPATH, By.XPATH,
"//div[@class='timeline-status-component-content']//img" "//div[@class='timeline-status-component-content']//img"
) )
WebDriverWait(driver, timeout=10).until( WebDriverWait(driver, timeout=10).until(
lambda d: d.execute_script("return arguments[0].complete", img) lambda d: d.execute_script("return arguments[0].complete", img)
) )
time.sleep(3) time.sleep(3)
${seleniumTakeScreenshot "\"/home/selenium/screenshot.png\""} ${seleniumTakeScreenshot "\"/home/selenium/screenshot.png\""}
${seleniumQuit}''; ${seleniumQuit}'';
seleniumScriptGetSrc = pkgs.writers.writePython3Bin "selenium-script-get-src" seleniumScriptGetSrc =
{ pkgs.writers.writePython3Bin "selenium-script-get-src"
libraries = with pkgs.python3Packages; [ selenium ]; {
} '' libraries = with pkgs.python3Packages; [ selenium ];
${seleniumImports} }
${seleniumSetup} ''
${seleniumPixelfedLogin} ${seleniumImports}
${seleniumSetup}
${seleniumPixelfedLogin}
img = driver.find_element( img = driver.find_element(
By.XPATH, By.XPATH,
"//div[@class='timeline-status-component-content']//img" "//div[@class='timeline-status-component-content']//img"
) )
# REVIEW: Need to wait for it to be loaded? # REVIEW: Need to wait for it to be loaded?
print(img.get_attribute('src')) print(img.get_attribute('src'))
${seleniumQuit}''; ${seleniumQuit}'';
in in
pkgs.nixosTest { pkgs.nixosTest {
name = "test-pixelfed-garage"; name = "test-pixelfed-garage";
nodes = { nodes = {
server = { config, ... }: { server =
{ config, ... }:
{
services = { services = {
xserver = { xserver = {
enable = true; enable = true;
displayManager.lightdm.enable = true; displayManager.lightdm.enable = true;
desktopManager.lxqt.enable = true; desktopManager.lxqt.enable = true;
};
displayManager.autoLogin = {
enable = true;
user = "selenium";
};
};
virtualisation.resolution = {
x = 1680;
y = 1050;
}; };
displayManager.autoLogin = { virtualisation = {
enable = true; memorySize = lib.mkVMOverride 8192;
user = "selenium"; cores = 8;
};
imports = with self.nixosModules; [
bleedingFediverse
fediversity
garage-vm
pixelfed-vm
];
# TODO: pair down
environment.systemPackages = with pkgs; [
python3
chromium
chromedriver
xh
seleniumScriptPostPicture
seleniumScriptGetSrc
helix
imagemagick
];
environment.variables = {
POST_MEDIA = ./fediversity.png;
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
## without this we get frivolous errors in the logs
MC_REGION = "garage";
};
# chrome does not like being run as root
users.users.selenium = {
isNormalUser = true;
}; };
}; };
virtualisation.resolution = { x = 1680; y = 1050; };
virtualisation = {
memorySize = lib.mkVMOverride 8192;
cores = 8;
};
imports = with self.nixosModules; [
bleedingFediverse
fediversity
garage-vm
pixelfed-vm
];
# TODO: pair down
environment.systemPackages = with pkgs; [
python3
chromium
chromedriver
xh
seleniumScriptPostPicture
seleniumScriptGetSrc
helix
imagemagick
];
environment.variables = {
POST_MEDIA = ./fediversity.png;
AWS_ACCESS_KEY_ID = config.services.garage.ensureKeys.pixelfed.id;
AWS_SECRET_ACCESS_KEY = config.services.garage.ensureKeys.pixelfed.secret;
## without this we get frivolous errors in the logs
MC_REGION = "garage";
};
# chrome does not like being run as root
users.users.selenium = {
isNormalUser = true;
};
};
}; };
testScript = { nodes, ... }: '' testScript =
import re { nodes, ... }:
''
import re
server.start() server.start()
with subtest("Pixelfed starts"): with subtest("Pixelfed starts"):
server.wait_for_unit("phpfpm-pixelfed.service") server.wait_for_unit("phpfpm-pixelfed.service")
with subtest("Account creation"): with subtest("Account creation"):
server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1") server.succeed("pixelfed-manage user:create --name=test --username=test --email=${email} --password=${password} --confirm_email=1")
# NOTE: This could in theory give a false positive if pixelfed changes it's # NOTE: This could in theory give a false positive if pixelfed changes it's
# colorscheme to include pure green. (see same problem in pixelfed-garage.nix). # colorscheme to include pure green. (see same problem in pixelfed-garage.nix).
# TODO: For instance: post a red image and check that the green pixel IS NOT # TODO: For instance: post a red image and check that the green pixel IS NOT
# there, then post a green image and check that the green pixel IS there. # there, then post a green image and check that the green pixel IS there.
with subtest("Image displays"): with subtest("Image displays"):
server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'") server.succeed("su - selenium -c 'selenium-script-post-picture ${email} ${password}'")
server.copy_from_vm("/home/selenium/screenshot.png", "") server.copy_from_vm("/home/selenium/screenshot.png", "")
displayed_colors = server.succeed("magick /home/selenium/screenshot.png -define histogram:unique-colors=true -format %c histogram:info:") displayed_colors = server.succeed("magick /home/selenium/screenshot.png -define histogram:unique-colors=true -format %c histogram:info:")
# check that the green image displayed somewhere # check that the green image displayed somewhere
image_check = re.match(".*#FF0500.*", displayed_colors, re.S) image_check = re.match(".*#FF0500.*", displayed_colors, re.S)
if image_check is None: if image_check is None:
raise Exception("cannot detect the uploaded image on pixelfed page.") raise Exception("cannot detect the uploaded image on pixelfed page.")
with subtest("access garage"): with subtest("access garage"):
server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY") server.succeed("mc alias set garage ${nodes.server.fediversity.internal.garage.api.url} --api s3v4 --path off $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY")
server.succeed("mc ls garage/pixelfed") server.succeed("mc ls garage/pixelfed")
with subtest("access image in garage"): with subtest("access image in garage"):
image = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'") image = server.succeed("mc find garage --regex '\\.png' --ignore '*_thumb.png'")
image = image.rstrip() image = image.rstrip()
if image == "": if image == "":
raise Exception("image posted to Pixelfed did not get stored in garage") raise Exception("image posted to Pixelfed did not get stored in garage")
server.succeed(f"mc cat {image} >/garage-image.png") server.succeed(f"mc cat {image} >/garage-image.png")
garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.png") garage_image_hash = server.succeed("identify -quiet -format '%#' /garage-image.png")
image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA") image_hash = server.succeed("identify -quiet -format '%#' $POST_MEDIA")
if garage_image_hash != image_hash: if garage_image_hash != image_hash:
raise Exception("image stored in garage did not match image uploaded") raise Exception("image stored in garage did not match image uploaded")
with subtest("Check that image comes from garage"): with subtest("Check that image comes from garage"):
src = server.succeed("su - selenium -c 'selenium-script-get-src ${email} ${password}'") src = server.succeed("su - selenium -c 'selenium-script-get-src ${email} ${password}'")
if not src.startswith("${nodes.server.fediversity.internal.garage.web.urlForBucket "pixelfed"}"): if not src.startswith("${nodes.server.fediversity.internal.garage.web.urlForBucket "pixelfed"}"):
raise Exception("image does not come from garage") raise Exception("image does not come from garage")
''; '';
} }

View file

@ -1,32 +1,42 @@
pkgs: test: pkgs: test:
let let
inherit (pkgs.lib) mapAttrsToList concatStringsSep genAttrs mkIf; inherit (pkgs.lib)
mapAttrsToList
concatStringsSep
genAttrs
mkIf
;
inherit (builtins) attrNames; inherit (builtins) attrNames;
interactiveConfig = ({ config, ... }: { interactiveConfig = (
# so we can run `nix shell nixpkgs#foo` on the machines { config, ... }:
nix.extraOptions = '' {
extra-experimental-features = nix-command flakes # so we can run `nix shell nixpkgs#foo` on the machines
''; nix.extraOptions = ''
extra-experimental-features = nix-command flakes
'';
# so we can ssh in and rebuild them # so we can ssh in and rebuild them
services.openssh = { services.openssh = {
enable = true; enable = true;
settings = { settings = {
PermitRootLogin = "yes"; PermitRootLogin = "yes";
PermitEmptyPasswords = "yes"; PermitEmptyPasswords = "yes";
UsePAM = false; UsePAM = false;
};
}; };
};
virtualisation = mkIf (config.networking.hostName == "jumphost") { virtualisation = mkIf (config.networking.hostName == "jumphost") {
forwardPorts = [{ forwardPorts = [
from = "host"; {
host.port = 2222; from = "host";
guest.port = 22; host.port = 2222;
}]; guest.port = 22;
}; }
}); ];
};
}
);
sshConfig = pkgs.writeText "ssh-config" '' sshConfig = pkgs.writeText "ssh-config" ''
Host * Host *
@ -50,10 +60,11 @@ let
# create an association array from machine names to the path to their # create an association array from machine names to the path to their
# configuration in the nix store # configuration in the nix store
declare -A configPaths=(${ declare -A configPaths=(${
concatStringsSep " " concatStringsSep " " (
(mapAttrsToList mapAttrsToList (
(n: v: ''["${n}"]="${v.system.build.toplevel}"'') n: v: ''["${n}"]="${v.system.build.toplevel}"''
rebuildableTest.driverInteractive.nodes) ) rebuildableTest.driverInteractive.nodes
)
}) })
rebuild_one() { rebuild_one() {
@ -113,37 +124,40 @@ let
# we're at it) # we're at it)
rebuildableTest = rebuildableTest =
let let
preOverride = pkgs.nixosTest (test // { preOverride = pkgs.nixosTest (
interactive = (test.interactive or { }) // { test
# no need to // with test.interactive.nodes here, since we are iterating // {
# over all of them, and adding back in the config via `imports` interactive = (test.interactive or { }) // {
nodes = genAttrs # no need to // with test.interactive.nodes here, since we are iterating
( # over all of them, and adding back in the config via `imports`
attrNames test.nodes or { } ++ nodes =
attrNames test.interactive.nodes or { } ++ genAttrs (attrNames test.nodes or { } ++ attrNames test.interactive.nodes or { } ++ [ "jumphost" ])
[ "jumphost" ] (n: {
) imports = [
(n: { (test.interactive.${n} or { })
imports = [ interactiveConfig
(test.interactive.${n} or { }) ];
interactiveConfig });
]; };
}); # override with test.passthru in case someone wants to overwrite us.
}; passthru = {
# override with test.passthru in case someone wants to overwrite us. inherit rebuildScript sshConfig;
passthru = { inherit rebuildScript sshConfig; } // (test.passthru or { }); } // (test.passthru or { });
}); }
);
in in
preOverride // { preOverride
// {
driverInteractive = preOverride.driverInteractive.overrideAttrs (old: { driverInteractive = preOverride.driverInteractive.overrideAttrs (old: {
# this comes from runCommand, not mkDerivation, so this is the only # this comes from runCommand, not mkDerivation, so this is the only
# hook we have to override # hook we have to override
buildCommand = old.buildCommand + '' buildCommand =
ln -s ${sshConfig} $out/ssh-config old.buildCommand
ln -s ${rebuildScript}/bin/rebuild $out/bin/rebuild + ''
''; ln -s ${sshConfig} $out/ssh-config
ln -s ${rebuildScript}/bin/rebuild $out/bin/rebuild
'';
}); });
}; };
in in
rebuildableTest rebuildableTest

View file

@ -1,4 +1,9 @@
{ lib, config, modulesPath, ... }: {
lib,
config,
modulesPath,
...
}:
let let
inherit (lib) mkVMOverride mapAttrs' filterAttrs; inherit (lib) mkVMOverride mapAttrs' filterAttrs;
@ -7,7 +12,8 @@ let
fedicfg = config.fediversity.internal.garage; fedicfg = config.fediversity.internal.garage;
in { in
{
imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ]; imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ];
services.nginx.virtualHosts = services.nginx.virtualHosts =
@ -16,9 +22,11 @@ in {
forceSSL = mkVMOverride false; forceSSL = mkVMOverride false;
enableACME = mkVMOverride false; enableACME = mkVMOverride false;
}; };
in mapAttrs' in
(bucket: _: {name = fedicfg.web.domainForBucket bucket; inherit value;}) mapAttrs' (bucket: _: {
(filterAttrs (_: {website, ...}: website) cfg.ensureBuckets); name = fedicfg.web.domainForBucket bucket;
inherit value;
}) (filterAttrs (_: { website, ... }: website) cfg.ensureBuckets);
virtualisation.diskSize = 2048; virtualisation.diskSize = 2048;
virtualisation.forwardPorts = [ virtualisation.forwardPorts = [

View file

@ -1,5 +1,6 @@
# customize nixos-rebuild build-vm to be a bit more convenient # customize nixos-rebuild build-vm to be a bit more convenient
{ pkgs, ... }: { { pkgs, ... }:
{
# let us log in # let us log in
users.mutableUsers = false; users.mutableUsers = false;
users.users.root.hashedPassword = ""; users.users.root.hashedPassword = "";
@ -34,7 +35,10 @@
# no graphics. see nixos-shell # no graphics. see nixos-shell
virtualisation = { virtualisation = {
graphics = false; graphics = false;
qemu.consoles = [ "tty0" "hvc0" ]; qemu.consoles = [
"tty0"
"hvc0"
];
qemu.options = [ qemu.options = [
"-serial null" "-serial null"
"-device virtio-serial" "-device virtio-serial"
@ -45,7 +49,10 @@
}; };
# we can't forward port 80 or 443, so let's run nginx on a different port # we can't forward port 80 or 443, so let's run nginx on a different port
networking.firewall.allowedTCPPorts = [ 8443 8080 ]; networking.firewall.allowedTCPPorts = [
8443
8080
];
services.nginx.defaultSSLListenPort = 8443; services.nginx.defaultSSLListenPort = 8443;
services.nginx.defaultHTTPListenPort = 8080; services.nginx.defaultHTTPListenPort = 8080;
virtualisation.forwardPorts = [ virtualisation.forwardPorts = [

View file

@ -1,4 +1,10 @@
{ modulesPath, lib, config, ... }: { {
modulesPath,
lib,
config,
...
}:
{
imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ]; imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ];

View file

@ -1,4 +1,5 @@
{ pkgs, modulesPath, ... }: { { pkgs, modulesPath, ... }:
{
imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ]; imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ];

View file

@ -1,9 +1,15 @@
{ pkgs, lib, modulesPath, ... }: {
pkgs,
lib,
modulesPath,
...
}:
let let
inherit (lib) mkVMOverride; inherit (lib) mkVMOverride;
in { in
{
imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ]; imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ];
fediversity = { fediversity = {