hans/Fediversity
1
0
Fork 0
forked from Fediversity/Fediversity
Code Pull requests Activity

first stab at a nixos test

Browse source 
for now, had to get rid of vmVariant. we can figure out how to add it
back when we understand how we should actually distinguish between
real machines and VMs
This commit is contained in:
Taeer Bar-Yam 2024-06-25 06:39:04 -04:00
parent 4e719da9d9
commit 693e21b1a8
9 changed files with 364 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
nixos.qcow2
result*
.direnv
.nixos-test-history

10
README.md
View file

@ -27,8 +27,11 @@ With the VM running, you can then access the apps on your local machine's web br
NOTE: it sometimes takes a while for the services to start up, and in the meantime you will get 502 Bad Gateway.
- Mastodon: <http://mastodon.localhost:55001>
- You can also create accounts on the machine itself by running `mastodon-tootctl accounts create test --email test@test.com --confirmed --approve`
- Mastodon: through the reverse proxy at <https://mastodon.localhost:8443> and directly at <http://mastodon.localhost:55001>
- You can create accounts on the machine itself by running `mastodon-tootctl accounts create test --email test@test.com --confirmed --approve`
- Account-related activities (logging in/out; preferences) can only be done on the insecure direct page <http://mastodon.localhost:55001>
- After you've logged in, you can go back to the secure page and you will remain logged in
- some operations may remove the port number from the URL. You'll have to add that back in manually
- PeerTube: <http://peertube.localhost:9000>
- The root account can be accessed with username "root". The password can be obtained by running the following command on the VM:
@ -51,6 +54,7 @@ NOTE: it sometimes takes a while for the services to start up, and in the meanti
- mastodon-web.service
- peertube.service
- the `garage` CLI command gives information about garage storage, but cannot be used to actually inspect the contents. use `mc` (minio) for that
- in the chromium devtools, you can go to the networking tab and change things like response headers in a way that persists through reloads. this is much faster iteration time if that's what you need to epxeriment with.
# questions
@ -77,5 +81,7 @@ When mastodon is running in production mode, we have a few problems:
- you have to click "accept the security risk"
- it takes a while for the webpage to come online. Until then you see "502 Bad Gateway"
- email sent from the mastodon instance (e.g. for account confirmation) should be accessible at <https://mastodon.localhost:55001/letter_opener>, but it's not working.
- mastodon is trying to fetch `missing.png` without ssl (`http://`). This isn't allowed, and i'm not sure why it's doing it.
- mastodon is trying to fetch `custom.css` from https://mastodon.localhost (no port), which is not the configured `LOCAL_DOMAIN`, so it's unclear why.

19
common.nix
View file

@ -44,5 +44,24 @@
"-device virtconsole,chardev=char0,nr=0"
];
};
# we can't forward port 80 or 443, so let's run nginx on a different port
networking.firewall.allowedTCPPorts = [ 8443 8080 ];
services.nginx.defaultSSLListenPort = 8443;
services.nginx.defaultHTTPListenPort = 8080;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 8080;
guest.port = 8080;
}
{
from = "host";
host.port = 8443;
guest.port = 8443;
}
];
};
}

11
flake.nix
View file

@ -11,6 +11,13 @@
pkgs = nixpkgs.legacyPackages.${system};
in {
nixosModules = {
mastodon = import ./mastodon.nix;
peertube = import ./peertube.nix;
pixelfed = import ./pixelfed.nix;
garage = import ./garage.nix;
};
nixosConfigurations = {
mastodon = nixpkgs.lib.nixosSystem {
inherit system;
@ -33,6 +40,10 @@
};
};
checks.${system} = {
mastodon-garage = import ./tests/mastodon-garage.nix { inherit pkgs self; };
};
devShells.${system}.default = pkgs.mkShell {
inputs = with pkgs; [
nil

30
garage.nix
View file

@ -124,21 +124,19 @@ in {
};
config = {
virtualisation.vmVariant = {
virtualisation.diskSize = 2048;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 3901;
guest.port = 3901;
}
{
from = "host";
host.port = 3902;
guest.port = 3902;
}
];
};
virtualisation.diskSize = 2048;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 3901;
guest.port = 3901;
}
{
from = "host";
host.port = 3902;
guest.port = 3902;
}
];
environment.systemPackages = [ pkgs.minio-client pkgs.awscli ];
@ -190,7 +188,7 @@ in {
${ensureBucketsScript}
${ensureKeysScript}
# garage doesn't like deleting keys that once existed
# garage doesn't like re-adding keys that once existed, so we can't delete / recreate it every time
# garage key delete ${snakeoil_key.id} --yes
'';
};

206
mastodon.nix
View file

@ -10,12 +10,12 @@ in
ensureBuckets = {
mastodon = {
website = true;
# corsRules = {
# enable = true;
# allowedHeaders = [ "*" ];
# allowedMethods = [ "GET" ];
# allowedOrigins = [ "*" ];
# };
corsRules = {
enable = true;
allowedHeaders = [ "*" ];
allowedMethods = [ "GET" ];
allowedOrigins = [ "*" ];
};
};
};
ensureKeys = {
@ -47,7 +47,7 @@ in
# but we want "<S3_BUCKET>.<S3_HOSTNAME>"
S3_ALIAS_HOST = "mastodon.web.garage.localhost:3902";
# XXX: I think we need to set up a proper CDN host
CDN_HOST = "mastodon.web.garage.localhost:3902";
# CDN_HOST = "mastodon.web.garage.localhost:3902";
# SEE: the last section in https://docs.joinmastodon.org/admin/optional/object-storage/
# TODO: can we set up ACLs with garage?
S3_PERMISSION = "";
@ -82,116 +82,118 @@ in
}
# VM setup
{
# these configurations only apply when producing a VM (e.g. nixos-rebuild build-vm)
virtualisation.vmVariant = { config, ... }: {
services.mastodon = {
# redirects to localhost, but allows it to have a proper domain name
localDomain = "mastodon.localhost";
services.mastodon = {
# redirects to localhost, but allows it to have a proper domain name
localDomain = "mastodon.localhost";
smtp = {
fromAddress = "mastodon@mastodon.localhost";
createLocally = false;
};
extraConfig = {
EMAIL_DOMAIN_ALLOWLIST = "example.com";
};
# from the documentation: recommended is the amount of your CPU cores minus one.
# but it also must be a positive integer
streamingProcesses = lib.max 1 (config.virtualisation.cores - 1);
smtp = {
fromAddress = "mastodon@mastodon.localhost";
createLocally = false;
};
security.acme = {
defaults = {
# invalid server; the systemd service will fail, and we won't get properly signed certificates
# but let's not spam the letsencrypt servers (and we don't own this domain anyways)
server = "https://127.0.0.1";
email = "none";
};
extraConfig = {
EMAIL_DOMAIN_ALLOWLIST = "example.com";
};
virtualisation.memorySize = 2048;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 44443;
guest.port = 443;
}
];
# from the documentation: recommended is the amount of your CPU cores minus one.
# but it also must be a positive integer
streamingProcesses = lib.max 1 (config.virtualisation.cores - 1);
};
security.acme = {
defaults = {
# invalid server; the systemd service will fail, and we won't get properly signed certificates
# but let's not spam the letsencrypt servers (and we don't own this domain anyways)
server = "https://127.0.0.1";
email = "none";
};
};
virtualisation.memorySize = 2048;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 44443;
guest.port = 443;
}
];
}
# mastodon development environment
{
networking.firewall.allowedTCPPorts = [ 55001 ];
virtualisation.vmVariant = { config, ... }: {
services.mastodon = {
# needed so we can directly access mastodon at port 55001
# otherwise, mastodon has to be accessed *from* port 443, which we can't do via port forwarding
enableUnixSocket = false;
extraConfig = {
RAILS_ENV = "development";
# to be accessible from outside the VM
BIND = "0.0.0.0";
# for letter_opener (still doesn't work though)
REMOTE_DEV = "true";
};
services.mastodon = {
# needed so we can directly access mastodon at port 55001
# otherwise, mastodon has to be accessed *from* port 443, which we can't do via port forwarding
enableUnixSocket = false;
extraConfig = {
RAILS_ENV = "development";
# to be accessible from outside the VM
BIND = "0.0.0.0";
# for letter_opener (still doesn't work though)
REMOTE_DEV = "true";
LOCAL_DOMAIN = "mastodon.localhost:8443";
};
};
# services.nginx.virtualHosts."${config.services.mastodon.localDomain}" = {
# extraConfig = ''
# add_header Content-Security-Policy 'base-uri 'none'; default-src 'none'; frame-ancestors 'none'; font-src 'self' http://mastodon.localhost:8443; img-src * https: data: blob: http://mastodon.localhost:8443; style-src 'self' http://mastodon.localhost:8443 'nonce-QvwdQ3lNRMmEcQnhZ22MAg=='; media-src 'self' https: data: http://mastodon.localhost:8443; frame-src 'self' https:; manifest-src 'self' http://mastodon.localhost:8443; form-action 'self'; child-src 'self' blob: http://mastodon.localhost:8443; worker-src 'self' blob: http://mastodon.localhost:8443; connect-src 'self' data: blob: http://mastodon.localhost:8443 http://mastodon.web.garage.localhost:3902 ws://mastodon.localhost:4000 ws://localhost:3035 http://localhost:3035; script-src 'self' 'unsafe-inline' 'unsafe-eval' http://mastodon.localhost:8443'
# '';
# };
# services.nginx.virtualHosts."${config.services.mastodon.localDomain}".locations."/sw.js" =
services.postgresql = {
enable = true;
ensureUsers = [
{
name = config.services.mastodon.database.user;
ensureClauses.createdb = true;
# ensurePermissions doesn't work anymore
# ensurePermissions = {
# "mastodon_development.*" = "ALL PRIVILEGES";
# "mastodon_test.*" = "ALL PRIVILEGES";
# }
}
];
# ensureDatabases = [ "mastodon_development_test" "mastodon_test" ];
};
# Currently, nixos seems to be able to create a single database per
# postgres user. This works for the production version of mastodon, which
# is what's packaged in nixpkgs. For development, we need two databases,
# mastodon_development and mastodon_test. This used to be possible with
# ensurePermissions, but that's broken and has been removed. Here I copy
# the mastodon-init-db script from upstream nixpkgs, but add the single
# line `rails db:setup`, which asks mastodon to create the postgres
# databases for us.
# FIXME: the commented out lines were breaking things, but presumably they're necessary for something.
# TODO: see if we can fix the upstream ensurePermissions stuff. See above for what that config would look like.
systemd.services.mastodon-init-db.script = lib.mkForce ''
result="$(psql -t --csv -c \
"select count(*) from pg_class c \
join pg_namespace s on s.oid = c.relnamespace \
where s.nspname not in ('pg_catalog', 'pg_toast', 'information_schema') \
and s.nspname not like 'pg_temp%';")" || error_code=$?
if [ "''${error_code:-0}" -ne 0 ]; then
echo "Failure checking if database is seeded. psql gave exit code $error_code"
exit "$error_code"
fi
if [ "$result" -eq 0 ]; then
echo "Seeding database"
rails db:setup
# SAFETY_ASSURED=1 rails db:schema:load
rails db:seed
else
# echo "Migrating database (this might be a noop)"
# rails db:migrate
fi
'';
virtualisation.forwardPorts = [
services.postgresql = {
enable = true;
ensureUsers = [
{
from = "host";
host.port = 55001;
guest.port = 55001;
name = config.services.mastodon.database.user;
ensureClauses.createdb = true;
# ensurePermissions doesn't work anymore
# ensurePermissions = {
# "mastodon_development.*" = "ALL PRIVILEGES";
# "mastodon_test.*" = "ALL PRIVILEGES";
# }
}
];
# ensureDatabases = [ "mastodon_development_test" "mastodon_test" ];
};
# Currently, nixos seems to be able to create a single database per
# postgres user. This works for the production version of mastodon, which
# is what's packaged in nixpkgs. For development, we need two databases,
# mastodon_development and mastodon_test. This used to be possible with
# ensurePermissions, but that's broken and has been removed. Here I copy
# the mastodon-init-db script from upstream nixpkgs, but add the single
# line `rails db:setup`, which asks mastodon to create the postgres
# databases for us.
# FIXME: the commented out lines were breaking things, but presumably they're necessary for something.
# TODO: see if we can fix the upstream ensurePermissions stuff. See above for what that config would look like.
systemd.services.mastodon-init-db.script = lib.mkForce ''
result="$(psql -t --csv -c \
"select count(*) from pg_class c \
join pg_namespace s on s.oid = c.relnamespace \
where s.nspname not in ('pg_catalog', 'pg_toast', 'information_schema') \
and s.nspname not like 'pg_temp%';")" || error_code=$?
if [ "''${error_code:-0}" -ne 0 ]; then
echo "Failure checking if database is seeded. psql gave exit code $error_code"
exit "$error_code"
fi
if [ "$result" -eq 0 ]; then
echo "Seeding database"
rails db:setup
# SAFETY_ASSURED=1 rails db:schema:load
rails db:seed
# else
# echo "Migrating database (this might be a noop)"
# rails db:migrate
fi
'';
virtualisation.forwardPorts = [
{
from = "host";
host.port = 55001;
guest.port = 55001;
}
];
}
]
]

BIN
tests/fediversity.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 5.5 KiB

58
tests/mastodon-garage.nix Normal file
View file

@ -0,0 +1,58 @@
{ pkgs, self }:
let
# python = pkgs.python310.withPackages (ps: with ps; [ requests aiokafka ]);
rebuildableTest = import ./rebuildableTest.nix pkgs;
in
rebuildableTest {
name = "test-mastodon-garage";
# skipLint = true;
# skipTypeCheck = true;
nodes = {
server = {
imports = [ self.nixosModules.garage self.nixosModules.mastodon ];
environment.systemPackages = with pkgs; [ toot ];
};
};
testScript = ''
import re
import time
server.start()
with subtest("Mastodon starts"):
server.wait_for_unit("mastodon-web.service")
# make sure mastodon is fully up and running before we interact with it
time.sleep(180)
with subtest("Account creation"):
account_creation_output = server.succeed("mastodon-tootctl accounts create test --email test@test.com --confirmed --approve")
password_re = re.compile('New password: (.*)')
password_match = password_re.match(account_creation_output)
assert password_match is not None
password = password_match.groups()[0]
with subtest("Log in with toot"):
# toot doesn't provide a way to just specify our login details as arguments, so we have to pretend we're typing them in at the prompt
server.send_chars("toot login_ctl\n")
time.sleep(0.2)
# Enter instance URL
server.send_chars("http://mastodon.localhost:55001\n")
time.sleep(0.2)
# Email
server.send_chars("test@test.com\n")
time.sleep(0.2)
# Password
server.send_chars(password + "\n")
with subtest("Post an image"):
server.succeed("toot post --media ${./fediversity.png}")
# TODO: I don't think there's a good way to test for whether the image visually shows up.
# we can test for CORS headers using curl / xh
# or **maybe** somehow read the javascript console?
'';
}

149
tests/rebuildableTest.nix Normal file
View file

@ -0,0 +1,149 @@
pkgs: test:
let
inherit (pkgs.lib) mapAttrsToList concatStringsSep genAttrs mkIf;
inherit (builtins) attrNames;
interactiveConfig = ({ config, ... }: {
# so we can run `nix shell nixpkgs#foo` on the machines
nix.extraOptions = ''
extra-experimental-features = nix-command flakes
'';
# so we can ssh in and rebuild them
services.openssh = {
enable = true;
settings = {
PermitRootLogin = "yes";
PermitEmptyPasswords = "yes";
UsePAM = "no";
};
};
virtualisation = mkIf (config.networking.hostName == "jumphost") {
forwardPorts = [{
from = "host";
host.port = 2222;
guest.port = 22;
}];
};
});
sshConfig = pkgs.writeText "ssh-config" ''
Host *
User root
StrictHostKeyChecking no
BatchMode yes
ConnectTimeout 20
UserKnownHostsFile=/dev/null
LogLevel Error # no "added to known hosts"
Host jumphost
Port 2222
HostName localhost
Host * !jumphost
ProxyJump jumphost
'';
# one should first start up the interactive test driver, then start the
# machines, then update the config, and then redeploy with the `rebuildScript`
# associated with the new config.
rebuildScript = pkgs.writeShellScriptBin "rebuild" ''
# create an association array from machine names to the path to their
# configuration in the nix store
declare -A configPaths=(${
concatStringsSep " "
(mapAttrsToList
(n: v: ''["${n}"]="${v.system.build.toplevel}"'')
rebuildableTest.driverInteractive.nodes)
})
rebuild_one() {
machine="$1"
echo "pushing new config to $machine"
if [ -z ''${configPaths[$machine]+x} ]; then
echo 'No machine '"$machine"' in this test.'
exit 1
fi
if ! ssh -F ${sshConfig} $machine true; then
echo 'Couldn'"'"'t connect to '"$machine"'. Make sure you'"'"'ve started it with `'"$machine"'.start()` in the test interactive driver.'
exit 1
fi
# taken from nixos-rebuild (we only want to do the activate part)
cmd=(
"systemd-run"
"-E" "LOCALE_ARCHIVE"
"--collect"
"--no-ask-password"
"--pty"
"--quiet"
"--same-dir"
"--service-type=exec"
"--unit=nixos-rebuild-switch-to-configuration"
"--wait"
"''${configPaths[$machine]}/bin/switch-to-configuration"
"test"
)
if ! ssh -F ${sshConfig} $machine "''${cmd[@]}"; then
echo "warning: error(s) occurred while switching to the new configuration"
exit 1
fi
}
if ! ssh -F ${sshConfig} jumphost true; then
echo 'Couldn'"'"'t connect to jump host. Make sure you are running driverInteractive, and that you'"'"'ve run `jumphost.start()` and `jumphost.forward_port(2222,22)`'
exit 1
fi
if [ -n "$1" ]; then
rebuild_one "$1"
else
for machine in ${concatStringsSep " " (attrNames rebuildableTest.driverInteractive.nodes)}; do
rebuild_one $machine
done
fi
'';
# NOTE: This is awkward because NixOS does not expose the module interface
# that is used to build tests. When we upstream this, we can build it into the
# system more naturally (and expose more of the interface to end users while
# we're at it)
rebuildableTest =
let
preOverride = pkgs.nixosTest (test // {
interactive = (test.interactive or { }) // {
# no need to // with test.interactive.nodes here, since we are iterating
# over all of them, and adding back in the config via `imports`
nodes = genAttrs
(
attrNames test.nodes or { } ++
attrNames test.interactive.nodes or { } ++
[ "jumphost" ]
)
(n: {
imports = [
(test.interactive.${n} or { })
interactiveConfig
];
});
};
# override with test.passthru in case someone wants to overwrite us.
passthru = { inherit rebuildScript sshConfig; } // (test.passthru or { });
});
in
preOverride // {
driverInteractive = preOverride.driverInteractive.overrideAttrs (old: {
# this comes from runCommand, not mkDerivation, so this is the only
# hook we have to override
buildCommand = old.buildCommand + ''
ln -s ${sshConfig} $out/ssh-config
ln -s ${rebuildScript}/bin/rebuild $out/bin/rebuild
'';
});
};
in
rebuildableTest