first stab at a nixos test

for now, had to get rid of vmVariant. we can figure out how to add it
back when we understand how we should actually distinguish between
real machines and VMs

.gitignore

@@ -1,4 +1,5 @@
 nixos.qcow2
 result*
 .direnv
+.nixos-test-history
 

README.md

@@ -27,8 +27,11 @@ With the VM running, you can then access the apps on your local machine's web br
 
 NOTE: it sometimes takes a while for the services to start up, and in the meantime you will get 502 Bad Gateway.
 
-- Mastodon: <http://mastodon.localhost:55001>
-  - You can also create accounts on the machine itself by running `mastodon-tootctl accounts create test --email test@test.com --confirmed --approve`
+- Mastodon: through the reverse proxy at <https://mastodon.localhost:8443> and directly at <http://mastodon.localhost:55001>
+  - You can create accounts on the machine itself by running `mastodon-tootctl accounts create test --email test@test.com --confirmed --approve`
+  - Account-related activities (logging in/out; preferences) can only be done on the insecure direct page <http://mastodon.localhost:55001>
+  - After you've logged in, you can go back to the secure page and you will remain logged in
+  - some operations may remove the port number from the URL. You'll have to add that back in manually
 
 - PeerTube: <http://peertube.localhost:9000>
   - The root account can be accessed with username "root". The password can be obtained by running the following command on the VM:
@@ -51,6 +54,7 @@ NOTE: it sometimes takes a while for the services to start up, and in the meanti
   - mastodon-web.service
   - peertube.service
 - the `garage` CLI command gives information about garage storage, but cannot be used to actually inspect the contents. use `mc` (minio) for that
+- in the chromium devtools, you can go to the networking tab and change things like response headers in a way that persists through reloads. this is much faster iteration time if that's what you need to epxeriment with.
 
 # questions
 
@@ -77,5 +81,7 @@ When mastodon is running in production mode, we have a few problems:
 - you have to click "accept the security risk"
 - it takes a while for the webpage to come online. Until then you see "502 Bad Gateway"
 - email sent from the mastodon instance (e.g. for account confirmation) should be accessible at <https://mastodon.localhost:55001/letter_opener>, but it's not working.
+- mastodon is trying to fetch `missing.png` without ssl (`http://`). This isn't allowed, and i'm not sure why it's doing it.
+- mastodon is trying to fetch `custom.css` from https://mastodon.localhost (no port), which is not the configured `LOCAL_DOMAIN`, so it's unclear why.
 
 

common.nix

@@ -44,5 +44,24 @@
         "-device virtconsole,chardev=char0,nr=0"
       ];
     };
+
+
+    # we can't forward port 80 or 443, so let's run nginx on a different port
+    networking.firewall.allowedTCPPorts = [ 8443 8080 ];
+    services.nginx.defaultSSLListenPort = 8443;
+    services.nginx.defaultHTTPListenPort = 8080;
+    virtualisation.forwardPorts = [
+      {
+        from = "host";
+        host.port = 8080;
+        guest.port = 8080;
+      }
+      {
+        from = "host";
+        host.port = 8443;
+        guest.port = 8443;
+      }
+    ];
+    
   };
 }

flake.nix

@@ -11,6 +11,13 @@
     pkgs = nixpkgs.legacyPackages.${system};
   in {
 
+    nixosModules = {
+      mastodon = import ./mastodon.nix;
+      peertube = import ./peertube.nix;
+      pixelfed = import ./pixelfed.nix;
+      garage = import ./garage.nix;
+    };
+
     nixosConfigurations = {
       mastodon = nixpkgs.lib.nixosSystem {
         inherit system;
@@ -33,6 +40,10 @@
       };
     };
 
+    checks.${system} = {
+      mastodon-garage = import ./tests/mastodon-garage.nix { inherit pkgs self; };
+    };
+
     devShells.${system}.default = pkgs.mkShell {
       inputs = with pkgs; [
         nil

garage.nix

@@ -124,21 +124,19 @@ in {
   };
 
   config = {
-    virtualisation.vmVariant = {
-      virtualisation.diskSize = 2048;
-      virtualisation.forwardPorts = [
-        {
-          from = "host";
-          host.port = 3901;
-          guest.port = 3901;
-        }
-        {
-          from = "host";
-          host.port = 3902;
-          guest.port = 3902;
-        }
-      ];
-    };
+    virtualisation.diskSize = 2048;
+    virtualisation.forwardPorts = [
+      {
+        from = "host";
+        host.port = 3901;
+        guest.port = 3901;
+      }
+      {
+        from = "host";
+        host.port = 3902;
+        guest.port = 3902;
+      }
+    ];
 
     environment.systemPackages = [ pkgs.minio-client pkgs.awscli ];
 
@@ -190,7 +188,7 @@ in {
         ${ensureBucketsScript}
         ${ensureKeysScript}
 
-        # garage doesn't like deleting keys that once existed
+        # garage doesn't like re-adding keys that once existed, so we can't delete / recreate it every time
         # garage key delete ${snakeoil_key.id} --yes
       '';
     };

mastodon.nix

@@ -10,12 +10,12 @@ in
       ensureBuckets = {
         mastodon = {
           website = true;
-          # corsRules = {
-          #   enable = true;
-          #   allowedHeaders = [ "*" ];
-          #   allowedMethods = [ "GET" ];
-          #   allowedOrigins = [ "*" ];
-          # };
+          corsRules = {
+            enable = true;
+            allowedHeaders = [ "*" ];
+            allowedMethods = [ "GET" ];
+            allowedOrigins = [ "*" ];
+          };
         };
       };
       ensureKeys = {
@@ -47,7 +47,7 @@ in
         # but we want "<S3_BUCKET>.<S3_HOSTNAME>"
         S3_ALIAS_HOST = "mastodon.web.garage.localhost:3902";
         # XXX: I think we need to set up a proper CDN host
-        CDN_HOST = "mastodon.web.garage.localhost:3902";
+        # CDN_HOST = "mastodon.web.garage.localhost:3902";
         # SEE: the last section in https://docs.joinmastodon.org/admin/optional/object-storage/
         # TODO: can we set up ACLs with garage?
         S3_PERMISSION = "";
@@ -82,116 +82,118 @@ in
   }
   # VM setup
   {
-    # these configurations only apply when producing a VM (e.g. nixos-rebuild build-vm)
-    virtualisation.vmVariant = { config, ... }: {
-      services.mastodon = {
-        # redirects to localhost, but allows it to have a proper domain name
-        localDomain = "mastodon.localhost";
+    services.mastodon = {
+      # redirects to localhost, but allows it to have a proper domain name
+      localDomain = "mastodon.localhost";
 
-        smtp = {
-          fromAddress = "mastodon@mastodon.localhost";
-          createLocally = false;
-        };
-
-        extraConfig = {
-          EMAIL_DOMAIN_ALLOWLIST = "example.com";
-        };
-
-        # from the documentation: recommended is the amount of your CPU cores minus one.
-        # but it also must be a positive integer
-        streamingProcesses = lib.max 1 (config.virtualisation.cores - 1);
+      smtp = {
+        fromAddress = "mastodon@mastodon.localhost";
+        createLocally = false;
       };
 
-      security.acme = {
-        defaults = {
-          # invalid server; the systemd service will fail, and we won't get properly signed certificates
-          # but let's not spam the letsencrypt servers (and we don't own this domain anyways)
-          server = "https://127.0.0.1";
-          email = "none";
-        };
+      extraConfig = {
+        EMAIL_DOMAIN_ALLOWLIST = "example.com";
       };
 
-      virtualisation.memorySize = 2048;
-      virtualisation.forwardPorts = [
-        {
-          from = "host";
-          host.port = 44443;
-          guest.port = 443;
-        }
-      ];
+      # from the documentation: recommended is the amount of your CPU cores minus one.
+      # but it also must be a positive integer
+      streamingProcesses = lib.max 1 (config.virtualisation.cores - 1);
     };
+
+    security.acme = {
+      defaults = {
+        # invalid server; the systemd service will fail, and we won't get properly signed certificates
+        # but let's not spam the letsencrypt servers (and we don't own this domain anyways)
+        server = "https://127.0.0.1";
+        email = "none";
+      };
+    };
+
+    virtualisation.memorySize = 2048;
+    virtualisation.forwardPorts = [
+      {
+        from = "host";
+        host.port = 44443;
+        guest.port = 443;
+      }
+    ];
   }
 
   # mastodon development environment
   {
     networking.firewall.allowedTCPPorts = [ 55001 ];
-    virtualisation.vmVariant = { config, ... }: {
-      services.mastodon = {
-        # needed so we can directly access mastodon at port 55001
-        # otherwise, mastodon has to be accessed *from* port 443, which we can't do via port forwarding
-        enableUnixSocket = false;
-        extraConfig = {
-          RAILS_ENV = "development";
-          # to be accessible from outside the VM
-          BIND = "0.0.0.0";
-          # for letter_opener (still doesn't work though)
-          REMOTE_DEV = "true";
-        };
+    services.mastodon = {
+      # needed so we can directly access mastodon at port 55001
+      # otherwise, mastodon has to be accessed *from* port 443, which we can't do via port forwarding
+      enableUnixSocket = false;
+      extraConfig = {
+        RAILS_ENV = "development";
+        # to be accessible from outside the VM
+        BIND = "0.0.0.0";
+        # for letter_opener (still doesn't work though)
+        REMOTE_DEV = "true";
+        LOCAL_DOMAIN = "mastodon.localhost:8443";
       };
+    };
+    # services.nginx.virtualHosts."${config.services.mastodon.localDomain}" = {
+    #   extraConfig = ''
+    #     add_header Content-Security-Policy 'base-uri 'none'; default-src 'none'; frame-ancestors 'none'; font-src 'self' http://mastodon.localhost:8443; img-src * https: data: blob: http://mastodon.localhost:8443; style-src 'self' http://mastodon.localhost:8443 'nonce-QvwdQ3lNRMmEcQnhZ22MAg=='; media-src 'self' https: data: http://mastodon.localhost:8443; frame-src 'self' https:; manifest-src 'self' http://mastodon.localhost:8443; form-action 'self'; child-src 'self' blob: http://mastodon.localhost:8443; worker-src 'self' blob: http://mastodon.localhost:8443; connect-src 'self' data: blob: http://mastodon.localhost:8443 http://mastodon.web.garage.localhost:3902 ws://mastodon.localhost:4000 ws://localhost:3035 http://localhost:3035; script-src 'self' 'unsafe-inline' 'unsafe-eval' http://mastodon.localhost:8443'
+    #   '';
+    # };
+    # services.nginx.virtualHosts."${config.services.mastodon.localDomain}".locations."/sw.js" = 
 
-      services.postgresql = {
-        enable = true;
-        ensureUsers = [
-          {
-            name = config.services.mastodon.database.user;
-            ensureClauses.createdb = true;
-            # ensurePermissions doesn't work anymore
-            # ensurePermissions = {
-            #   "mastodon_development.*" = "ALL PRIVILEGES";
-            #   "mastodon_test.*" = "ALL PRIVILEGES";
-            # }
-          }
-        ];
-        # ensureDatabases = [ "mastodon_development_test" "mastodon_test" ];
-      };
-
-      # Currently, nixos seems to be able to create a single database per
-      # postgres user. This works for the production version of mastodon, which
-      # is what's packaged in nixpkgs. For development, we need two databases,
-      # mastodon_development and mastodon_test. This used to be possible with
-      # ensurePermissions, but that's broken and has been removed. Here I copy
-      # the mastodon-init-db script from upstream nixpkgs, but add the single
-      # line `rails db:setup`, which asks mastodon to create the postgres
-      # databases for us.
-      # FIXME: the commented out lines were breaking things, but presumably they're necessary for something.
-      # TODO: see if we can fix the upstream ensurePermissions stuff. See above for what that config would look like.
-      systemd.services.mastodon-init-db.script = lib.mkForce ''
-        result="$(psql -t --csv -c \
-            "select count(*) from pg_class c \
-            join pg_namespace s on s.oid = c.relnamespace \
-            where s.nspname not in ('pg_catalog', 'pg_toast', 'information_schema') \
-            and s.nspname not like 'pg_temp%';")" || error_code=$?
-        if [ "''${error_code:-0}" -ne 0 ]; then
-          echo "Failure checking if database is seeded. psql gave exit code $error_code"
-          exit "$error_code"
-        fi
-        if [ "$result" -eq 0 ]; then
-          echo "Seeding database"
-          rails db:setup
-          # SAFETY_ASSURED=1 rails db:schema:load
-          rails db:seed
-        else
-          # echo "Migrating database (this might be a noop)"
-          # rails db:migrate
-        fi
-      '';
-      virtualisation.forwardPorts = [
+    services.postgresql = {
+      enable = true;
+      ensureUsers = [
         {
-          from = "host";
-          host.port = 55001;
-          guest.port = 55001;
+          name = config.services.mastodon.database.user;
+          ensureClauses.createdb = true;
+          # ensurePermissions doesn't work anymore
+          # ensurePermissions = {
+          #   "mastodon_development.*" = "ALL PRIVILEGES";
+          #   "mastodon_test.*" = "ALL PRIVILEGES";
+          # }
         }
       ];
+      # ensureDatabases = [ "mastodon_development_test" "mastodon_test" ];
     };
+
+    # Currently, nixos seems to be able to create a single database per
+    # postgres user. This works for the production version of mastodon, which
+    # is what's packaged in nixpkgs. For development, we need two databases,
+    # mastodon_development and mastodon_test. This used to be possible with
+    # ensurePermissions, but that's broken and has been removed. Here I copy
+    # the mastodon-init-db script from upstream nixpkgs, but add the single
+    # line `rails db:setup`, which asks mastodon to create the postgres
+    # databases for us.
+    # FIXME: the commented out lines were breaking things, but presumably they're necessary for something.
+    # TODO: see if we can fix the upstream ensurePermissions stuff. See above for what that config would look like.
+    systemd.services.mastodon-init-db.script = lib.mkForce ''
+      result="$(psql -t --csv -c \
+          "select count(*) from pg_class c \
+          join pg_namespace s on s.oid = c.relnamespace \
+          where s.nspname not in ('pg_catalog', 'pg_toast', 'information_schema') \
+          and s.nspname not like 'pg_temp%';")" || error_code=$?
+      if [ "''${error_code:-0}" -ne 0 ]; then
+        echo "Failure checking if database is seeded. psql gave exit code $error_code"
+        exit "$error_code"
+      fi
+      if [ "$result" -eq 0 ]; then
+        echo "Seeding database"
+        rails db:setup
+        # SAFETY_ASSURED=1 rails db:schema:load
+        rails db:seed
+      # else
+        # echo "Migrating database (this might be a noop)"
+        # rails db:migrate
+      fi
+    '';
+    virtualisation.forwardPorts = [
+      {
+        from = "host";
+        host.port = 55001;
+        guest.port = 55001;
+      }
+    ];
   }
-] 
+]

tests/mastodon-garage.nix

@@ -0,0 +1,58 @@
+{ pkgs, self }:
+let
+  # python = pkgs.python310.withPackages (ps: with ps; [ requests aiokafka ]);
+  rebuildableTest = import ./rebuildableTest.nix pkgs;
+in
+rebuildableTest {
+  name = "test-mastodon-garage";
+
+  # skipLint = true;
+  # skipTypeCheck = true;
+
+  nodes = {
+    server = {
+      imports = [ self.nixosModules.garage self.nixosModules.mastodon ];
+      environment.systemPackages = with pkgs; [ toot ];
+    };
+  };
+
+  testScript = ''
+    import re
+    import time
+
+    server.start()
+
+    with subtest("Mastodon starts"):
+      server.wait_for_unit("mastodon-web.service")
+
+    # make sure mastodon is fully up and running before we interact with it
+    time.sleep(180)
+
+    with subtest("Account creation"):
+      account_creation_output = server.succeed("mastodon-tootctl accounts create test --email test@test.com --confirmed --approve")
+      password_re = re.compile('New password: (.*)')
+      password_match = password_re.match(account_creation_output)
+      assert password_match is not None
+      password = password_match.groups()[0]
+
+    with subtest("Log in with toot"):
+      # toot doesn't provide a way to just specify our login details as arguments, so we have to pretend we're typing them in at the prompt
+      server.send_chars("toot login_ctl\n")
+      time.sleep(0.2)
+      # Enter instance URL
+      server.send_chars("http://mastodon.localhost:55001\n")
+      time.sleep(0.2)
+      # Email
+      server.send_chars("test@test.com\n")
+      time.sleep(0.2)
+      # Password
+      server.send_chars(password + "\n")
+
+    with subtest("Post an image"):
+      server.succeed("toot post --media ${./fediversity.png}")
+
+      # TODO: I don't think there's a good way to test for whether the image visually shows up. 
+      # we can test for CORS headers using curl / xh
+      # or **maybe** somehow read the javascript console?
+  '';
+}

tests/rebuildableTest.nix

@@ -0,0 +1,149 @@
+pkgs: test:
+let
+  inherit (pkgs.lib) mapAttrsToList concatStringsSep genAttrs mkIf;
+  inherit (builtins) attrNames;
+
+  interactiveConfig = ({ config, ... }: {
+    # so we can run `nix shell nixpkgs#foo` on the machines
+    nix.extraOptions = ''
+      extra-experimental-features = nix-command flakes
+    '';
+
+    # so we can ssh in and rebuild them
+    services.openssh = {
+      enable = true;
+      settings = {
+        PermitRootLogin = "yes";
+        PermitEmptyPasswords = "yes";
+        UsePAM = "no";
+      };
+    };
+
+    virtualisation = mkIf (config.networking.hostName == "jumphost") {
+      forwardPorts = [{
+        from = "host";
+        host.port = 2222;
+        guest.port = 22;
+      }];
+    };
+  });
+
+  sshConfig = pkgs.writeText "ssh-config" ''
+    Host *
+      User root
+      StrictHostKeyChecking no
+      BatchMode yes
+      ConnectTimeout 20
+      UserKnownHostsFile=/dev/null
+      LogLevel Error # no "added to known hosts"
+    Host jumphost
+      Port 2222
+      HostName localhost
+    Host * !jumphost
+      ProxyJump jumphost
+  '';
+
+  # one should first start up the interactive test driver, then start the
+  # machines, then update the config, and then redeploy with the `rebuildScript`
+  # associated with the new config.
+  rebuildScript = pkgs.writeShellScriptBin "rebuild" ''
+    # create an association array from machine names to the path to their
+    # configuration in the nix store
+    declare -A configPaths=(${
+      concatStringsSep " "
+        (mapAttrsToList
+          (n: v: ''["${n}"]="${v.system.build.toplevel}"'')
+          rebuildableTest.driverInteractive.nodes)
+    })
+
+    rebuild_one() {
+      machine="$1"
+      echo "pushing new config to $machine"
+
+      if [ -z ''${configPaths[$machine]+x} ]; then
+        echo 'No machine '"$machine"' in this test.'
+        exit 1
+      fi
+
+      if ! ssh -F ${sshConfig} $machine true; then
+        echo 'Couldn'"'"'t connect to '"$machine"'. Make sure you'"'"'ve started it with `'"$machine"'.start()` in the test interactive driver.'
+        exit 1
+      fi
+
+      # taken from nixos-rebuild (we only want to do the activate part)
+       cmd=(
+          "systemd-run"
+          "-E" "LOCALE_ARCHIVE"
+          "--collect"
+          "--no-ask-password"
+          "--pty"
+          "--quiet"
+          "--same-dir"
+          "--service-type=exec"
+          "--unit=nixos-rebuild-switch-to-configuration"
+          "--wait"
+          "''${configPaths[$machine]}/bin/switch-to-configuration"
+          "test"
+      )
+
+
+      if ! ssh -F ${sshConfig} $machine "''${cmd[@]}"; then
+          echo "warning: error(s) occurred while switching to the new configuration"
+          exit 1
+      fi
+    }
+
+    if ! ssh -F ${sshConfig} jumphost true; then
+      echo 'Couldn'"'"'t connect to jump host. Make sure you are running driverInteractive, and that you'"'"'ve run `jumphost.start()` and `jumphost.forward_port(2222,22)`'
+      exit 1
+    fi
+
+    if [ -n "$1" ]; then
+      rebuild_one "$1"
+    else
+      for machine in ${concatStringsSep " " (attrNames rebuildableTest.driverInteractive.nodes)}; do
+        rebuild_one $machine
+      done
+    fi
+  '';
+
+  # NOTE: This is awkward because NixOS does not expose the module interface
+  # that is used to build tests. When we upstream this, we can build it into the
+  # system more naturally (and expose more of the interface to end users while
+  # we're at it)
+  rebuildableTest =
+    let
+      preOverride = pkgs.nixosTest (test // {
+        interactive = (test.interactive or { }) // {
+          # no need to // with test.interactive.nodes here, since we are iterating
+          # over all of them, and adding back in the config via `imports`
+          nodes = genAttrs
+            (
+              attrNames test.nodes or { } ++
+                attrNames test.interactive.nodes or { } ++
+                [ "jumphost" ]
+            )
+            (n: {
+              imports = [
+                (test.interactive.${n} or { })
+                interactiveConfig
+              ];
+            });
+        };
+        # override with test.passthru in case someone wants to overwrite us.
+        passthru = { inherit rebuildScript sshConfig; } // (test.passthru or { });
+      });
+    in
+    preOverride // {
+      driverInteractive = preOverride.driverInteractive.overrideAttrs (old: {
+        # this comes from runCommand, not mkDerivation, so this is the only
+        # hook we have to override
+        buildCommand = old.buildCommand + ''
+          ln -s ${sshConfig} $out/ssh-config
+          ln -s ${rebuildScript}/bin/rebuild $out/bin/rebuild
+        '';
+      });
+    };
+in
+rebuildableTest
+