From 2c7e3603b84d17ac7e73b33937f1b6a695b2ce06 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Fri, 24 May 2024 19:02:12 -0400 Subject: [PATCH] better documentation and readme --- README.md | 69 ++++++++++++++++++++++++---------------------------- common.nix | 21 +++++++++------- flake.nix | 4 +-- garage.nix | 24 +++++++++++++----- mastodon.nix | 12 ++++++++- peertube.nix | 4 ++- pixelfed.nix | 52 ++++++++++++++++++++++++++++++++++++++- 7 files changed, 129 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index bea1a92..d8f6210 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Fediverse VMs -This repo is, for now, an attempt to familiarize myself with NixOS options for Fediverse applications, and build up a configuration layer that will set most of the relevant options for you (in a semi-opinionated way) given some high-level configuration. This is in the same vein as [nixos-mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver). +This repo is, for now, an attempt to familiarize myself with NixOS options for Fediverse applications, and build up a configuration layer that will set most of the relevant options for you (in a semi-opinionated way) given some high-level configuration. The goal is something in the same vein as [nixos-mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver) but for fediversity. Eventually, this will be tailored to high-throughput multi-machine setups. For now, it's just a small configuration to run in VMs. @@ -19,13 +19,16 @@ and then run it with ./result/bin/run-nixos-vm ``` -You can then access the apps on your local machine (using the magic of port forwarding) at the following addresses +After the machine boots, you should be dropped into a root shell. + +Note that state will be persisted in the `nixos.cqow2` file. Delete that and restart the VM to reset the state. + +With the VM running, you can then access the apps on your local machine's web browser (using the magic of port forwarding) at the following addresses + +NOTE: it sometimes takes a while for the services to start up, and in the meantime you will get 502 Bad Gateway. - Mastodon: - - You will have to "accept the security risk" - - It may take a minute for the webpage to come online. Until then you will see "502 Bad Gateway" - - (NOTE: currently broken) email sent from the mastodon instance (e.g. for setting up an account) will be accessible at - - You can also create accounts on the machine itself by running `mastodon-tootctl accounts create --email --confirmed --approve` + - You can also create accounts on the machine itself by running `mastodon-tootctl accounts create test --email test@test.com --confirmed --approve` - PeerTube: - The root account can be accessed with username "root". The password can be obtained by running the following command on the VM: @@ -34,53 +37,45 @@ You can then access the apps on your local machine (using the magic of port forw ``` - Creating other accounts has to be enabled via the admin interface. `Administration > Configuration > Basic > Enable Signup` or just add an account directly from `Administration > Create user`. But functionality can also be tested from the root account. +- Pixelfed: + - Account creation via the web interface won't work until we figure out email + - For now, they can be created on the VM command line + ```bash + pixelfed-manage user:create --name=test --username=test --email=test@test.com --password=testtest --confirm_email=1 + ``` + ## debugging notes - it is sometimes useful to `cat result/bin/run-nixos-vm` to see what's really going on (e.g. which ports are getting forwarded) - relevant systemd services: - mastodon-web.service - peertube.service - - unclear yet which pixelfed services are useful -- you can ssh to the machine using `ssh -p 2222 root@localhost` - -# TODOs - -- [ ] set up a domain name and a DNS service so we can do deploy this to an actual machine -- [ ] set up an email service -- [ ] add logging - - [ ] errors / logs - - [ ] performance -- [ ] switch to garage / s3 storage - - SEE: https://docs.joinmastodon.org/admin/optional/object-storage/ -- [ ] decouple the postgres database from this machine -- [ ] test with high use / throughput -- [ ] configure scaling behaviour - - SEE: https://docs.joinmastodon.org/admin/scaling/ -- [ ] remove the need for "accept security risk" dialogue if possible -- [ ] development environment does not work seamlessly. -- [x] don't require proxy server - - either forward 443 directly, or get mastodon to accept connections on a different port (maybe 3000? see development environment documentation) -- [ ] get letter_opener working -- [ ] share resources (e.g. s3 storage) between the services -- [ ] get garage running on another machine - - [ ] get garage replication running (multiple machines) -- [ ] some way of declaratively defining users? -- [ ] shared users between fediverse services -- [ ] s3 cache server (SEE: https://docs.joinpeertube.org/maintain/remote-storage) -- [ ] is "s3" the right term, given that it's not an open protocol? +- the `garage` CLI command gives information about garage storage, but cannot be used to actually inspect the contents. use `mc` (minio) for that # questions - what is meant to be shared between instances? - this is relevant to the security model. If garage is being shared between instances, we have to be careful having configurations depend on each other. - -- we want to be able to migrate user's data. s3 migration is not supported by peertube. what do? (SEE: https://docs.joinpeertube.org/maintain/remote-storage) + - they are to be shared, BUT the user will have no direct control over configuration. # resources - Tutorial for setting up better logging: https://krisztianfekete.org/self-hosting-mastodon-on-nixos-a-proof-of-concept/ -- Setting up development environment: https://docs.joinmastodon.org/dev/setup/ +- Setting up mastodon development environment: https://docs.joinmastodon.org/dev/setup/ - Tutorial for PeerTube that doesn't use `createLocally`: https://nixos.wiki/wiki/PeerTube - garage settings for specific apps: https://garagehq.deuxfleurs.fr/documentation/connect/apps/ + +- pixelfed has terrible / mostly non-existent documentation) + +- for when we start worry about scaling up: https://docs.joinmastodon.org/admin/scaling/ + +# notes + +When mastodon is running in production mode, we have a few problems: +- you have to click "accept the security risk" +- it takes a while for the webpage to come online. Until then you see "502 Bad Gateway" +- email sent from the mastodon instance (e.g. for account confirmation) should be accessible at , but it's not working. + + diff --git a/common.nix b/common.nix index d9674e8..150afaa 100644 --- a/common.nix +++ b/common.nix @@ -14,9 +14,19 @@ # automatically log in services.getty.autologinUser = "root"; - + services.getty.helpLine = '' + Type `C-a c` to access the qemu console + Type `C-a x` to quit + ''; # access to convenient things - environment.systemPackages = with pkgs; [ w3m python3 ]; + environment.systemPackages = with pkgs; [ + w3m + python3 + xterm # for `resize` + ]; + environment.loginShellInit = '' + eval "$(resize)" + ''; nix.extraOptions = '' extra-experimental-features = nix-command flakes ''; @@ -32,13 +42,6 @@ "-mon chardev=char0,mode=readline" "-device virtconsole,chardev=char0,nr=0" ]; - forwardPorts = [ - { - from = "host"; - host.port = 2222; - guest.port = 22; - } - ]; }; }; } diff --git a/flake.nix b/flake.nix index be5a1f2..9a6e011 100644 --- a/flake.nix +++ b/flake.nix @@ -24,12 +24,12 @@ pixelfed = nixpkgs.lib.nixosSystem { inherit system; - modules = [ ./common.nix ./pixelfed.nix ]; + modules = [ ./common.nix ./pixelfed.nix ./garage.nix ]; }; all = nixpkgs.lib.nixosSystem { inherit system; - modules = [ ./common.nix ./mastodon.nix ./peertube.nix ./pixelfed.nix ]; + modules = [ ./common.nix ./mastodon.nix ./peertube.nix ./pixelfed.nix ./garage.nix ]; }; }; diff --git a/garage.nix b/garage.nix index b3f2652..e7ad923 100644 --- a/garage.nix +++ b/garage.nix @@ -1,6 +1,6 @@ let # generate one using openssl (somehow) - # XXX: when importing, garage tells you importing is only meant for keys previously generated by garage. is it okay to generate them using openssl? + # XXX: when importing, garage tells you importing is only meant for keys previously generated by garage. is it okay to generate them using openssl? it seems to work fine snakeoil_key = { id = "GK22a15201acacbd51cd43e327"; secret = "82b2b4cbef27bf8917b350d5b10a87c92fa9c8b13a415aeeea49726cf335d74e"; @@ -21,6 +21,7 @@ in type = types.bool; default = false; }; + # I think setting corsRules should allow another website to show images from your bucket corsRules = { enable = mkEnableOption "CORS Rules"; allowedHeaders = mkOption { @@ -48,10 +49,10 @@ in # TODO: these should be managed as secrets, not in the nix store options = { id = mkOption { - type = types.string; + type = types.str; }; secret = mkOption { - type = types.string; + type = types.str; }; # TODO: assert at least one of these is true ensureAccess = mkOption { @@ -95,6 +96,7 @@ in } ]; }; + environment.systemPackages = [ pkgs.minio-client pkgs.awscli ]; networking.firewall.allowedTCPPorts = [ 3901 3902 ]; @@ -127,30 +129,37 @@ in sleep 3 # XXX: this is very sensitive to being a single instance - # (bare minimum to get garage up and running) + # (doing the bare minimum to get garage up and running) # also, it's crazy that we have to parse command output like this + # TODO: talk to garage maintainer about making this nicer to work with in Nix + # before I do that though, I should figure out how setting it up across multiple machines will work GARAGE_ID=$(garage node id 2>/dev/null | perl -ne '/(.*)@.*/ && print $1') garage layout assign -z g1 -c 1G $GARAGE_ID LAYOUT_VER=$(garage layout show | perl -ne '/Current cluster layout version: (\d*)/ && print $1') garage layout apply --version $((LAYOUT_VER + 1)) # XXX: this is a hack because we want to write to the buckets here but we're not guaranteed any access keys + # TODO: generate this key here rather than using a well-known key garage key import --yes -n tmp ${snakeoil_key.id} ${snakeoil_key.secret} export AWS_ACCESS_KEY_ID=${snakeoil_key.id}; export AWS_SECRET_ACCESS_KEY=${snakeoil_key.secret}; ${ lib.concatStringsSep "\n" (lib.mapAttrsToList (bucket: { website, aliases, corsRules }: '' - garage bucket create ${bucket} - # XXX: should this --deny the website if `website` is false? + # garage bucket info tells us if the bucket already exists + garage bucket info ${bucket} || garage bucket create ${bucket} + + # TODO: should this --deny the website if `website` is false? ${lib.optionalString website '' garage bucket website --allow ${bucket} ''} + ${lib.concatStringsSep "\n" (map (alias: '' garage bucket alias ${bucket} ${alias} '') aliases)} ${lib.optionalString corsRules.enable '' + # TODO: can i turn this whole thing into one builtins.toJSON? export CORS=${lib.concatStrings [ "'" ''{"CORSRules":[{'' @@ -160,6 +169,7 @@ in ''}]}'' "'" ]} + garage bucket allow --read --write --owner ${bucket} --key tmp aws --endpoint http://s3.garage.localhost:3900 s3api put-bucket-cors --bucket ${bucket} --cors-configuration $CORS garage bucket deny --read --write --owner ${bucket} --key tmp @@ -176,6 +186,8 @@ in } '') config.services.garage.ensureKeys) } + + garage key delete ${snakeoil_key.id} --yes ''; }; }; diff --git a/mastodon.nix b/mastodon.nix index ac43ea9..43e724f 100644 --- a/mastodon.nix +++ b/mastodon.nix @@ -8,7 +8,15 @@ in { # garage setup services.garage = { ensureBuckets = { - mastodon = { website = true; }; + mastodon = { + website = true; + # corsRules = { + # enable = true; + # allowedHeaders = [ "*" ]; + # allowedMethods = [ "GET" ]; + # allowedOrigins = [ "*" ]; + # }; + }; }; ensureKeys = { mastodon = { @@ -38,6 +46,8 @@ in # by default it tries to use "/" # but we want "." S3_ALIAS_HOST = "mastodon.web.garage.localhost:3902"; + # XXX: I think we need to set up a proper CDN host + CDN_HOST = "mastodon.web.garage.localhost:3902"; # SEE: the last section in https://docs.joinmastodon.org/admin/optional/object-storage/ # TODO: can we set up ACLs with garage? S3_PERMISSION = ""; diff --git a/peertube.nix b/peertube.nix index 777a726..4e641b0 100644 --- a/peertube.nix +++ b/peertube.nix @@ -11,6 +11,7 @@ in ensureBuckets = { peertube-videos = { website = true; + # TODO: these are too broad, after getting everything works narrow it down to the domain we actually want corsRules = { enable = true; allowedHeaders = [ "*" ]; @@ -18,6 +19,7 @@ in allowedOrigins = [ "*" ]; }; }; + # TODO: these are too broad, after getting everything works narrow it down to the domain we actually want peertube-playlists = { website = true; corsRules = { @@ -81,7 +83,7 @@ in AWS_ACCESS_KEY_ID=${snakeoil_key.id} AWS_SECRET_ACCESS_KEY=${snakeoil_key.secret} ''; - # these configurations only apply when producing a VM (e.g. nixos-rebuild build-vm) + virtualisation.vmVariant = { config, ... }: { services.peertube = { enable = true; diff --git a/pixelfed.nix b/pixelfed.nix index 186fc8b..9a64071 100644 --- a/pixelfed.nix +++ b/pixelfed.nix @@ -1,9 +1,57 @@ +let + snakeoil_key = { + id = "GKb5615457d44214411e673b7b"; + secret = "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in { config, lib, pkgs, ... }: { + + services.garage = { + ensureBuckets = { + pixelfed = { + website = true; + # TODO: these are too broad, after getting everything works narrow it down to the domain we actually want + corsRules = { + enable = true; + allowedHeaders = [ "*" ]; + allowedMethods = [ "GET" ]; + allowedOrigins = [ "*" ]; + }; + }; + }; + ensureKeys = { + pixelfed = { + inherit (snakeoil_key) id secret; + ensureAccess = { + pixelfed = { + read = true; + write = true; + owner = true; + }; + }; + }; + }; + }; + + # TODO: factor these out so we're only defining e.g. s3.garage.localhost and port 3900 in one place + services.pixelfed.settings = { + FILESYSTEM_CLOUD = "s3"; + PF_ENABLE_CLOUD = true; + AWS_ACCESS_KEY_ID = snakeoil_key.id; + AWS_SECRET_ACCESS_KEY = snakeoil_key.secret; + AWS_DEFAULT_REGION = "garage"; + AWS_URL = "http://pixelfed.s3.garage.localhost:3900"; + AWS_BUCKET = "pixelfed"; + AWS_ENDPOINT = "http://s3.garage.localhost:3900"; + AWS_USE_PATH_STYLE_ENDPOINT = false; + }; + virtualisation.vmVariant = { networking.firewall.allowedTCPPorts = [ 80 ]; services.pixelfed = { enable = true; domain = "pixelfed.localhost"; + # TODO: secrets management! secretFile = pkgs.writeText "secrets.env" '' APP_KEY=adKK9EcY8Hcj3PLU7rzG9rJ6KKTOtYfA ''; @@ -11,9 +59,11 @@ OPEN_REGISTRATION = true; FORCE_HTTPS_URLS = false; }; - # TODO: I feel like this should have an `enable` option and be configured via `services.nginx` rather than mirroring those options here + # I feel like this should have an `enable` option and be configured via `services.nginx` rather than mirroring those options here + # TODO: If that indeed makes sense, upstream it. nginx = {}; }; + virtualisation.memorySize = 2048; virtualisation.forwardPorts = [ { from = "host";