From 5fd1e115a009bc1b5844db07c547e56a69ef5935 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Wed, 27 Mar 2024 05:42:11 -0400 Subject: [PATCH] basic s3 garage setup for mastodon it's still having trouble fetching stored images for some reason --- README.md | 8 ++++++ flake.nix | 2 +- garage.nix | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ mastodon.nix | 28 ++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 garage.nix diff --git a/README.md b/README.md index b111967..acf1ba9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ You can then access the apps on your local machine (using the magic of port forw - You will have to "accept the security risk" - It may take a minute for the webpage to come online. Until then you will see "502 Bad Gateway" - (NOTE: currently broken) email sent from the mastodon instance (e.g. for setting up an account) will be accessible at + - You can also create accounts on the machine itself by running `mastodon-tootctl accounts create --email --confirmed --approve` - PeerTube: - The root account can be accessed with username "root". The password can be obtained by running the following command on the VM: @@ -61,6 +62,13 @@ You can then access the apps on your local machine (using the magic of port forw - either forward 443 directly, or get mastodon to accept connections on a different port (maybe 3000? see development environment documentation) - [ ] get letter_opener working - [ ] share resources (e.g. s3 storage) between the services +- [ ] get garage running on another machine + - [ ] get garage replication running (multiple machines) + +# questions + +- what is meant to be shared between instances? + - this is relevant to the security model. If garage is being shared between instances, we have to be careful having configurations depend on each other. # resources diff --git a/flake.nix b/flake.nix index d381e12..88c269e 100644 --- a/flake.nix +++ b/flake.nix @@ -14,7 +14,7 @@ nixosConfigurations = { mastodon = nixpkgs.lib.nixosSystem { inherit system; - modules = [ ./common.nix ./mastodon.nix ]; + modules = [ ./common.nix ./mastodon.nix ./garage.nix ]; }; peertube = nixpkgs.lib.nixosSystem { diff --git a/garage.nix b/garage.nix new file mode 100644 index 0000000..16e2adb --- /dev/null +++ b/garage.nix @@ -0,0 +1,74 @@ +let + # generate one using openssl (somehow) + # XXX: when importing, garage tells you importing is only meant for keys previously generated by garage. is it okay to generate them using openssl? + snakeoil_key = { + id = "GK3515373e4c851ebaad366558"; + secret = "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; +in +# TODO: expand to a multi-machine setup +{ config, lib, pkgs, ... }: { + # add in options to ensure creation of buckets and keys + + config = { + virtualisation.vmVariant = { + virtualisation.diskSize = 2048; + virtualisation.forwardPorts = [ + { + from = "host"; + host.port = 3901; + guest.port = 3901; + } + { + from = "host"; + host.port = 3902; + guest.port = 3902; + } + ]; + }; + environment.systemPackages = [ pkgs.minio-client ]; + + networking.firewall.allowedTCPPorts = [ 3901 3902 ]; + services.garage = { + enable = true; + package = pkgs.garage_0_9; + settings = { + replication_mode = "none"; + # TODO: use a secret file + rpc_secret = "d576c4478cc7d0d94cfc127138cbb82018b0155c037d1c827dfb6c36be5f6625"; + # TODO: why does this have to be set? is there not a sensible default? + rpc_bind_addr = "[::]:3901"; + rpc_public_addr = "[::1]:3901"; + s3_api.api_bind_addr = "[::]:3900"; + s3_web.bind_addr = "[::]:3902"; + s3_web.root_domain = ".web.garage.localhost"; + index = "index.html"; + + s3_api.s3_region = "garage"; + s3_api.root_domain = ".s3.garage.localhost"; + }; + }; + systemd.services.ensure-garage = { + after = [ "garage.service" ]; + wantedBy = [ "garage.service" ]; + path = [ config.services.garage.package pkgs.perl ]; + script = '' + set -xeuo pipefail + # give garage time to start up + sleep 3 + # XXX: this is very sensitive to being a single instance + # (bare minimum to get garage up and running) + # also, it's crazy that we have to parse command output like this + GARAGE_ID=$(garage node id 2>/dev/null | perl -ne '/(.*)@.*/ && print $1') + garage layout assign -z g1 -c 1G $GARAGE_ID + LAYOUT_VER=$(garage layout show | perl -ne '/Current cluster layout version: (\d*)/ && print $1') + garage layout apply --version $((LAYOUT_VER + 1)) + + garage bucket create mastodon + garage key import --yes -n mastodon "${snakeoil_key.id}" "${snakeoil_key.secret}" + garage bucket allow --read --write mastodon --key mastodon + garage bucket website --allow mastodon + ''; + }; + }; +} diff --git a/mastodon.nix b/mastodon.nix index 7fb1fe4..0db7738 100644 --- a/mastodon.nix +++ b/mastodon.nix @@ -1,4 +1,32 @@ +let + snakeoil_key = { + id = "GK3515373e4c851ebaad366558"; + secret = "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; +in { config, lib, pkgs, ... }: lib.mkMerge [ + { # garage setup + services.mastodon = { + extraConfig = { + S3_ENABLED = "true"; + S3_ENDPOINT = "http://s3.garage.localhost:3900"; + S3_REGION = "garage"; + S3_BUCKET = "mastodon"; + # use . + S3_OVERRIDE_PATH_STLE = "true"; + AWS_ACCESS_KEY_ID = snakeoil_key.id; + AWS_SECRET_ACCESS_KEY = snakeoil_key.secret; + S3_PROTOCOL = "http"; + S3_HOSTNAME = "web.garage.localhost:3902"; + # by default it tries to use "/" + # but we want "." + S3_ALIAS_HOST = "mastodon.web.garage.localhost:3902"; + # SEE: the last section in https://docs.joinmastodon.org/admin/optional/object-storage/ + # TODO: can we set up ACLs with garage? + S3_PERMISSION = ""; + }; + }; + } # mastodon setup { # open up access to the mastodon web interface