deploy garage

Signed-off-by: Kiara Grouwstra <kiara@procolix.eu>

TODO.md

@@ -1,3 +1,2 @@
-- deploy garage machine
 - port tests over from nixops
 - host supplies function to data model allocating available resources to demand

deployment/check/data-model-basic/nixosTest.nix

@@ -21,7 +21,6 @@ in
           default = mkOption {
             type = config.deployment-type;
             default = config.environments.default.deployment {
-              deployment-name = "default";
               configuration = config."example-configuration";
             };
           };

deployment/check/data-model-nixops4/data-model.nix

@@ -58,7 +58,6 @@ in
     options.default = lib.mkOption {
       type = config.deployment-type;
       default = config.environments.default.deployment {
-        deployment-name = "default";
         configuration = config."example-configuration";
       };
     };

deployment/check/data-model-ssh/data-model.nix

@@ -57,7 +57,6 @@ in
       default = mkOption {
         type = config.deployment-type;
         default = config.environments.default.deployment {
-          deployment-name = "default";
           configuration = config."example-configuration";
         };
       };

deployment/check/data-model-tf-proxmox/setups/template.nix

@@ -59,11 +59,10 @@ in
       default = mkOption {
         type = config.deployment-type;
         default = config.environments.default.deployment {
-          deployment-name = "default";
           # normally our template is distinct, but our test cannot download build deps due to sandboxing
           configuration = config."example-configuration";
-          };
         };
+      };
       deploy = mkOption {
         default = config.default.tf-proxmox-template.run;
       };

deployment/check/data-model-tf-proxmox/setups/vm.nix

@@ -86,10 +86,9 @@ in
       default = mkOption {
         type = config.deployment-type;
         default = config.environments.default.deployment {
-          deployment-name = "default";
           configuration = config."example-configuration";
-          };
         };
+      };
       deploy = mkOption {
         default = config.default.tf-proxmox-vm.run;
       };

deployment/check/data-model-tf/data-model.nix

@@ -57,10 +57,9 @@ in
       default = mkOption {
         type = config.deployment-type;
         default = config.environments.default.deployment {
-          deployment-name = "default";
           configuration = config."example-configuration";
-          };
         };
+      };
       deploy = mkOption {
         default = config.default.tf-host.run;
       };

deployment/data-model.nix

@@ -34,12 +34,32 @@ let
     )
   );
   deployment-type = attrTag (pkgs.callPackage ./run { inherit inputs sources; });
+  env-output = submodule {
+    options = {
+      deployments = mkOption {
+        type = deployment-type;
+      };
+      ancilliaryRequests = mkOption {
+        type = submodule {
+          options = {
+            garage = mkOption {
+              type = types.listOf types.unspecified;
+            };
+          };
+        };
+        default = { };
+      };
+    };
+  };
 in
 {
   options = {
     deployment-type = mkOption {
       default = deployment-type;
     };
+    env-output = mkOption {
+      default = env-output;
+    };
     resources = mkOption {
       description = "Collection of deployment resources that can be required by applications and policed by hosting providers";
       type = attrsOf (
@@ -158,6 +178,7 @@ in
                   lib.mapAttrs (_name: resource: mkOption { type = submodule resource.policy; }) config.resources
                 )
               );
+              default = { };
             };
             implementation = mkOption {
               description = "Mapping of resources required by applications to available resources; the result can be deployed";
@@ -170,14 +191,15 @@ in
                 input-type = submodule {
                   options = {
                     deployment-name = mkOption {
-                      type = types.str;
+                      type = types.listOf types.str;
+                      default = [ "default" ];
                     };
                     required-resources = mkOption {
                       type = attrsOf application-resources;
                     };
                   };
                 };
-                output-type = deployment-type;
+                output-type = env-output;
                 implementation = environment.config.implementation;
               };
             };
@@ -188,14 +210,15 @@ in
                 input-type = submodule {
                   options = {
                     deployment-name = mkOption {
-                      type = types.str;
+                      type = types.listOf types.str;
+                      default = [ "default" ];
                     };
                     configuration = mkOption {
                       type = config.configuration;
                     };
                   };
                 };
-                output-type = deployment-type;
+                output-type = env-output;
                 implementation =
                   {
                     deployment-name,

deployment/fediversity/resources/garage/default.nix

@@ -90,6 +90,7 @@ in
       {
         _class = "fediversity-resource-policy";
         options = {
+          enable = lib.mkEnableOption "S3-compatible storage provider Garage";
           rpc = {
             port = mkOption {
               type = types.int;
@@ -135,198 +136,223 @@ in
           resource-type = types.submodule {
             options = {
               garageSide = mkOption {
-                # type = types.unspecified;
-                # default = { };
                 type = types.listOf types.unspecified;
                 default = [ ];
               };
               applicationSide = mkOption {
-                # type = types.attrsOf types.unspecified;
-                # default = { };
                 type = types.listOf types.unspecified;
                 default = [ ];
               };
+              mainConfig = mkOption {
+                type = types.unspecified;
+                default = { };
+              };
             };
           };
-          apply =
+          apply = requests: {
-            requests:
+            applicationSide = map (req: req.nixos-configuration config) requests;
-            let
+
-              applicationSide = map (req: req.nixos-configuration config) requests;
+            mainConfig = nixos: {
-              garageSide =
+              config = {
-                let
+                environment.systemPackages = [
-                  # generate one using openssl (somehow)
+                  pkgs.minio-client
-                  # XXX: when importing, garage tells you importing is only meant for keys previously generated by garage. is it okay to generate them using openssl? it seems to work fine
+                  pkgs.awscli
-                  snakeoil_key = {
+                ];
-                    id = "GK22a15201acacbd51cd43e327";
+                ## REVIEW: Do we want to reverse proxy the RPC and API ports? In fact,
-                    secret = "82b2b4cbef27bf8917b350d5b10a87c92fa9c8b13a415aeeea49726cf335d74e";
+                ## shouldn't we just get rid of RPC at all, we're not using it.
+                networking.firewall.allowedTCPPorts = [
+                  80
+                  443
+                  config.api.port
+                  config.rpc.port
+                ];
+                services.garage = {
+                  enable = true;
+                  package = pkgs.garage_0_9;
+                  settings = {
+                    replication_mode = "none";
+                    # TODO: use a secret file
+                    rpc_secret = "d576c4478cc7d0d94cfc127138cbb82018b0155c037d1c827dfb6c36be5f6625";
+                    # TODO: why does this have to be set? is there not a sensible default?
+                    rpc_bind_addr = "[::]:${toString config.rpc.port}";
+                    rpc_public_addr = "[::1]:${toString config.rpc.port}";
+                    s3_api.api_bind_addr = "[::]:${toString config.api.port}";
+                    s3_web.bind_addr = "[::]:${toString config.web.internalPort}";
+                    s3_web.root_domain = ".${config.web.rootDomain}";
+                    index = "index.html";
+                    s3_api.s3_region = "garage";
+                    s3_api.root_domain = ".${config.api.domain}";
                   };
-                  inherit (builtins) toString;
+                };
-                  inherit (lib) optionalString concatStringsSep mkIf;
+                services.nginx.enable = true;
-                  inherit (lib.strings) escapeShellArg;
+                systemd.services.ensure-garage = {
-                  inherit (lib.attrsets) filterAttrs mapAttrs';
+                  after = [ "garage.service" ];
-                  concatMapAttrs = scriptFn: attrset: concatStringsSep "\n" (lib.mapAttrsToList scriptFn attrset);
+                  wantedBy = [ "garage.service" ];
-                  ensureAccessScriptFn =
+                  serviceConfig.Type = "oneshot";
-                    key: bucket:
+                  path = [
-                    {
+                    nixos.config.services.garage.package
-                      read,
+                    pkgs.perl
-                      write,
+                    pkgs.awscli
-                      owner,
+                  ];
-                    }:
+                };
-                    ''
+
-                      garage bucket allow ${optionalString read "--read"} ${optionalString write "--write"} ${optionalString owner "--owner"} \
+              };
-                        ${escapeShellArg bucket} --key ${escapeShellArg key}
+            };
+
+            garageSide =
+              let
+                # generate one using openssl (somehow)
+                # XXX: when importing, garage tells you importing is only meant for keys previously generated by garage. is it okay to generate them using openssl? it seems to work fine
+                snakeoil_key = {
+                  id = "GK22a15201acacbd51cd43e327";
+                  secret = "82b2b4cbef27bf8917b350d5b10a87c92fa9c8b13a415aeeea49726cf335d74e";
+                };
+                inherit (lib)
+                  optionals
+                  optionalString
+                  concatStringsSep
+                  lists
+                  strings
+                  attrsets
+                  ;
+                inherit (strings) escapeShellArg;
+                inherit (attrsets) filterAttrs mapAttrs';
+                inherit (lists) map;
+                concatMapAttrs = scriptFn: attrset: concatStringsSep "\n" (lib.mapAttrsToList scriptFn attrset);
+                ensureAccessScriptFn =
+                  key: bucket:
+                  {
+                    read,
+                    write,
+                    owner,
+                  }:
+                  ''
+                    garage bucket allow ${optionalString read "--read"} ${optionalString write "--write"} ${optionalString owner "--owner"} \
+                      ${escapeShellArg bucket} --key ${escapeShellArg key}
+                  '';
+                ensureKeyScriptFn =
+                  key:
+                  {
+                    s3AccessKeyFile,
+                    s3SecretKeyFile,
+                    ensureAccess,
+                  }:
+                  ''
+                    ## FIXME: Check whether the key exist and skip this step if that is the case. Get rid of this `|| :`
+                    garage key import --yes -n ${escapeShellArg key} $(cat ${escapeShellArg s3AccessKeyFile}) $(cat ${escapeShellArg s3SecretKeyFile}) || :
+                    ${concatMapAttrs (ensureAccessScriptFn key) ensureAccess}
+                  '';
+                ensureBucketScriptFn =
+                  cfg: bucket:
+                  {
+                    website,
+                    aliases,
+                    corsRules,
+                  }:
+                  let
+                    bucketArg = escapeShellArg bucket;
+                    corsRulesJSON = escapeShellArg (
+                      builtins.toJSON {
+                        CORSRules = [
+                          {
+                            AllowedHeaders = corsRules.allowedHeaders;
+                            AllowedMethods = corsRules.allowedMethods;
+                            AllowedOrigins = corsRules.allowedOrigins;
+                          }
+                        ];
+                      }
+                    );
+                  in
+                  ''
+                    # garage bucket info tells us if the bucket already exists
+                    garage bucket info ${bucketArg} || garage bucket create ${bucketArg}
+                    # TODO: should this --deny the website if `website` is false?
+                    ${optionalString website ''
+                      garage bucket website --allow ${bucketArg}
+                    ''}
+                    ${concatStringsSep "\n" (
+                      map (alias: ''
+                        garage bucket alias ${bucketArg} ${escapeShellArg alias}
+                      '') aliases
+                    )}
+                    ${optionalString corsRules.enable ''
+                      garage bucket allow --read --write --owner ${bucketArg} --key tmp
+                      # TODO: endpoint-url should not be hard-coded
+                      aws --region ${cfg.settings.s3_api.s3_region} --endpoint-url ${config.api.url} s3api put-bucket-cors --bucket ${bucketArg} --cors-configuration ${corsRulesJSON}
+                      garage bucket deny --read --write --owner ${bucketArg} --key tmp
+                    ''}
+                  '';
+                value = {
+                  forceSSL = true;
+                  enableACME = true;
+                  locations."/" = {
+                    proxyPass = "http://localhost:3902";
+                    extraConfig = ''
+                      ## copied from https://garagehq.deuxfleurs.fr/documentation/cookbook/reverse-proxy/
+                      proxy_set_header Host $host;
+                      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                      # Disable buffering to a temporary file.
+                      proxy_max_temp_file_size 0;
+                      ## NOTE: This page suggests many more options for the object storage
+                      ## proxy. We should take a look.
+                      ## https://docs.joinmastodon.org/admin/optional/object-storage-proxy/
                     '';
-                  ensureKeyScriptFn =
+                  };
-                    key:
+                };
-                    {
+                ensureBucketsScript =
-                      s3AccessKeyFile,
+                  cfg:
-                      s3SecretKeyFile,
+                  lib.concatStringsSep "\n" (
-                      ensureAccess,
+                    map ({ ensureBuckets, ... }: concatMapAttrs (ensureBucketScriptFn cfg) ensureBuckets) requests
-                    }:
+                  );
-                    ''
+                ensureKeysScript = lib.concatStringsSep "\n" (
-                      ## FIXME: Check whether the key exist and skip this step if that is the case. Get rid of this `|| :`
+                  map ({ ensureKeys, ... }: concatMapAttrs ensureKeyScriptFn ensureKeys) requests
-                      garage key import --yes -n ${escapeShellArg key} $(cat ${escapeShellArg s3AccessKeyFile}) $(cat ${escapeShellArg s3SecretKeyFile}) || :
+                );
-                      ${concatMapAttrs (ensureAccessScriptFn key) ensureAccess}
+                baseCfg = nixos: {
-                    '';
+                  config = {
-                  ensureBucketScriptFn =
+                    systemd.services.ensure-garage = {
-                    bucket:
+                      after = [ "garage.service" ];
-                    {
+                      wantedBy = [ "garage.service" ];
-                      website,
+                      serviceConfig.Type = "oneshot";
-                      aliases,
+                      path = [
-                      corsRules,
+                        nixos.config.services.garage.package
-                    }:
+                        pkgs.perl
-                    let
+                        pkgs.awscli
-                      bucketArg = escapeShellArg bucket;
+                      ];
-                      corsRulesJSON = escapeShellArg (
+                      script = ''
-                        builtins.toJSON {
+                        set -xeuo pipefail
-                          CORSRules = [
+                        # Give Garage time to start up by waiting until somethings speaks HTTP
-                            {
+                        # behind Garage's API URL.
-                              AllowedHeaders = corsRules.allowedHeaders;
+                        until ${pkgs.curl}/bin/curl -sio /dev/null ${config.api.url}; do sleep 1; done
-                              AllowedMethods = corsRules.allowedMethods;
+                        # XXX: this is very sensitive to being a single instance
-                              AllowedOrigins = corsRules.allowedOrigins;
+                        # (doing the bare minimum to get garage up and running)
-                            }
+                        # also, it's crazy that we have to parse command output like this
-                          ];
+                        # TODO: talk to garage maintainer about making this nicer to work with in Nix
-                        }
+                        # before I do that though, I should figure out how setting it up across multiple machines will work
-                      );
+                        GARAGE_ID=$(garage node id 2>/dev/null | perl -ne '/(.*)@.*/ && print $1')
-                    in
+                        garage layout assign -z g1 -c 1G $GARAGE_ID
-                    ''
+                        LAYOUT_VER=$(garage layout show | perl -ne '/Current cluster layout version: (\d*)/ && print $1')
-                      # garage bucket info tells us if the bucket already exists
+                        garage layout apply --version $((LAYOUT_VER + 1))
-                      garage bucket info ${bucketArg} || garage bucket create ${bucketArg}
+                        # XXX: this is a hack because we want to write to the buckets here but we're not guaranteed any access keys
-                      # TODO: should this --deny the website if `website` is false?
+                        # TODO: generate this key here rather than using a well-known key
-                      ${optionalString website ''
+                        # TODO: if the key already exists, we get an error; hacked with this `|| :` which needs to be removed
-                        garage bucket website --allow ${bucketArg}
+                        garage key import --yes -n tmp ${snakeoil_key.id} ${snakeoil_key.secret} || :
-                      ''}
+                        export AWS_ACCESS_KEY_ID=${snakeoil_key.id};
-                      ${concatStringsSep "\n" (
+                        export AWS_SECRET_ACCESS_KEY=${snakeoil_key.secret};
-                        map (alias: ''
+                        ${ensureBucketsScript nixos.config.services.garage}
-                          garage bucket alias ${bucketArg} ${escapeShellArg alias}
+                        ${ensureKeysScript}
-                        '') aliases
+                        # garage doesn't like re-adding keys that once existed, so we can't delete / recreate it every time
-                      )}
+                        # garage key delete ${snakeoil_key.id} --yes
-                      ${optionalString corsRules.enable ''
-                        garage bucket allow --read --write --owner ${bucketArg} --key tmp
-                        # TODO: endpoint-url should not be hard-coded
-                        aws --region ${config.settings.s3_api.s3_region} --endpoint-url ${config.api.url} s3api put-bucket-cors --bucket ${bucketArg} --cors-configuration ${corsRulesJSON}
-                        garage bucket deny --read --write --owner ${bucketArg} --key tmp
-                      ''}
-                    '';
-                  value = {
-                    forceSSL = true;
-                    enableACME = true;
-                    locations."/" = {
-                      proxyPass = "http://localhost:3902";
-                      extraConfig = ''
-                        ## copied from https://garagehq.deuxfleurs.fr/documentation/cookbook/reverse-proxy/
-                        proxy_set_header Host $host;
-                        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-                        # Disable buffering to a temporary file.
-                        proxy_max_temp_file_size 0;
-                        ## NOTE: This page suggests many more options for the object storage
-                        ## proxy. We should take a look.
-                        ## https://docs.joinmastodon.org/admin/optional/object-storage-proxy/
                       '';
                     };
                   };
-                  ensureBucketsScript = lib.concatStringsSep "\n" (lib.lists.map (
+                };
-                    { ensureBuckets }: concatMapAttrs ensureBucketScriptFn ensureBuckets
+              in
-                  )) requests;
+              optionals config.enable (
-                  ensureKeysScript = lib.concatStringsSep "\n" (lib.lists.map (
-                    { ensureKeys }: concatMapAttrs ensureKeyScriptFn ensureKeys
-                  )) requests;
-                  baseCfg = {
-                    config = mkIf config.enable {
-                      environment.systemPackages = [
-                        pkgs.minio-client
-                        pkgs.awscli
-                      ];
-                      ## REVIEW: Do we want to reverse proxy the RPC and API ports? In fact,
-                      ## shouldn't we just get rid of RPC at all, we're not using it.
-                      networking.firewall.allowedTCPPorts = [
-                        80
-                        443
-                        config.api.port
-                        config.rpc.port
-                      ];
-                      services.garage = {
-                        enable = true;
-                        package = pkgs.garage_0_9;
-                        settings = {
-                          replication_mode = "none";
-                          # TODO: use a secret file
-                          rpc_secret = "d576c4478cc7d0d94cfc127138cbb82018b0155c037d1c827dfb6c36be5f6625";
-                          # TODO: why does this have to be set? is there not a sensible default?
-                          rpc_bind_addr = "[::]:${toString config.rpc.port}";
-                          rpc_public_addr = "[::1]:${toString config.rpc.port}";
-                          s3_api.api_bind_addr = "[::]:${toString config.api.port}";
-                          s3_web.bind_addr = "[::]:${toString config.web.internalPort}";
-                          s3_web.root_domain = ".${config.web.rootDomain}";
-                          index = "index.html";
-                          s3_api.s3_region = "garage";
-                          s3_api.root_domain = ".${config.api.domain}";
-                        };
-                      };
-                      services.nginx.enable = true;
-                      systemd.services.ensure-garage = {
-                        after = [ "garage.service" ];
-                        wantedBy = [ "garage.service" ];
-                        serviceConfig.Type = "oneshot";
-                        path = [
-                          config.package
-                          pkgs.perl
-                          pkgs.awscli
-                        ];
-                        script = ''
-                          set -xeuo pipefail
-                          # Give Garage time to start up by waiting until somethings speaks HTTP
-                          # behind Garage's API URL.
-                          until ${pkgs.curl}/bin/curl -sio /dev/null ${config.api.url}; do sleep 1; done
-                          # XXX: this is very sensitive to being a single instance
-                          # (doing the bare minimum to get garage up and running)
-                          # also, it's crazy that we have to parse command output like this
-                          # TODO: talk to garage maintainer about making this nicer to work with in Nix
-                          # before I do that though, I should figure out how setting it up across multiple machines will work
-                          GARAGE_ID=$(garage node id 2>/dev/null | perl -ne '/(.*)@.*/ && print $1')
-                          garage layout assign -z g1 -c 1G $GARAGE_ID
-                          LAYOUT_VER=$(garage layout show | perl -ne '/Current cluster layout version: (\d*)/ && print $1')
-                          garage layout apply --version $((LAYOUT_VER + 1))
-                          # XXX: this is a hack because we want to write to the buckets here but we're not guaranteed any access keys
-                          # TODO: generate this key here rather than using a well-known key
-                          # TODO: if the key already exists, we get an error; hacked with this `|| :` which needs to be removed
-                          garage key import --yes -n tmp ${snakeoil_key.id} ${snakeoil_key.secret} || :
-                          export AWS_ACCESS_KEY_ID=${snakeoil_key.id};
-                          export AWS_SECRET_ACCESS_KEY=${snakeoil_key.secret};
-                          ${ensureBucketsScript}
-                          ${ensureKeysScript}
-                          # garage doesn't like re-adding keys that once existed, so we can't delete / recreate it every time
-                          # garage key delete ${snakeoil_key.id} --yes
-                        '';
-                      };
-                    };
-                  };
-                in
                 [
                   baseCfg
                 ]
-                ++ (lib.lists.map (
+                ++ (map (
-                  { ensureBuckets }:
+                  { ensureBuckets, ... }:
                   {
                     ## Create a proxy from <bucket>.web.garage.<domain> to localhost:3902 for
                     ## each bucket that has `website = true`.
@@ -335,11 +361,9 @@ in
                       inherit value;
                     }) (filterAttrs (_: { website, ... }: website) ensureBuckets);
                   }
-                ) requests);
+                ) requests)
-            in
+              );
-            {
+          };
-              inherit garageSide applicationSide;
-            };
         };
       };
   };

deployment/fediversity/ssh-host.nix

@@ -1,97 +0,0 @@
-{
-  system,
-  nodeName,
-  network,
-  sources ? import ../../npins,
-  conf ? { },
-  ...
-}@args:
-let
-  inherit (sources) nixpkgs;
-  pkgs = import nixpkgs { inherit system; };
-  inherit (pkgs) lib;
-  pathToRoot = builtins.path {
-    path = ../..;
-    name = "root";
-  };
-  sshOpts = [ ];
-in
-(pkgs.callPackage ../utils.nix { }).evalModel (
-  { config, modulesPath, ... }:
-  {
-    imports = [
-      # "${modulesPath}/profiles/minimal.nix"
-      ./resources
-      ./applications
-      ./configurations.nix
-      ../../infra/common/options.nix
-      # ../../infra/common/proxmox-qemu-vm.nix
-    ];
-    options = {
-      # inherit (config) default-configuration;
-      # inherit (conf) default-configuration;
-      # conf = lib.mkOption {
-      #   default = conf;
-      # };
-      default = lib.mkOption {
-        type = config.deployment-type;
-        default = config.environments.default.deployment {
-          deployment-name = "default";
-          configuration = config."default-configuration";
-        };
-      };
-    };
-    config = lib.mkMerge [
-      {
-        environments.default = environment: {
-          resources."external".garage = { };
-          resources."fediversity".nixos-module = { };
-          resources."fixed-host".network = network;
-          resources."age".secrets = { };
-          implementation =
-            {
-              required-resources,
-              deployment-name,
-              ...
-            }:
-            let
-              garage = environment.config.resources."external".garage.process required-resources;
-              # TODO use garage.garageSide
-            in
-            # builtins.trace required-resources.peertube.resources.peertube-bucket
-            {
-              ssh-host = {
-                nixos-configuration = {
-                  imports =
-                    [
-                      ../../infra/common/nixos
-                      "${sources.disko}/module.nix"
-                      "${modulesPath}/profiles/qemu-guest.nix"
-                      (environment.config.resources."fixed-host".network.process required-resources)
-                      (environment.config.resources."age".secrets.process required-resources)
-                    ]
-                    ++ (environment.config.resources."fediversity".nixos-module.process required-resources)
-                    ++ garage.applicationSide;
-                };
-                inherit system;
-                ssh = {
-                  username = "root";
-                  host = nodeName;
-                  key-file = null;
-                  inherit sshOpts;
-                };
-                caller = "deployment/fediversity/ssh-host.nix";
-                inherit args deployment-name;
-                root-path = pathToRoot;
-              };
-            };
-        };
-      }
-      conf
-      # splice global config into apps using it
-      {
-        default-configuration.applications.pixelfed = { inherit (conf.default-configuration) initialUser; };
-      }
-    ];
-  }
-)

deployment/fediversity/ssh-hosts.nix

@@ -1,23 +1,40 @@
 {
   system,
   host-mapping,
+  ancilliary,
   sources ? import ../../npins,
   conf ? { },
   ...
-}:
+}@args:
 let
   inherit (sources) nixpkgs;
   pkgs = import nixpkgs { inherit system; };
   inherit (pkgs) lib;
   inherit (lib) types;
-  pathToRoot = builtins.path {
+  inherit (pkgs.callPackage ../utils.nix { }) getSomeAttrs evalModel;
+  caller = "deployment/fediversity/ssh-hosts.nix";
+  root-path = builtins.path {
     path = ../..;
     name = "root";
   };
   sshOpts = [ ];
+  username = "root";
+  key-file = null;
+  apps = lib.attrNames host-mapping;
+  nodes = lib.attrNames ancilliary ++ apps;
+  hosts = ancilliary // host-mapping;
+  resources =
+    {
+      "external".garage.enable = true;
+      "fediversity".nixos-module = { };
+      "age".secrets = { };
+    }
+    // lib.mapAttrs (_: host: {
+      network = (import ../../machines/operator/${host}).fediversityVm;
+    }) hosts;
 in
-(pkgs.callPackage ../utils.nix { }).evalModel (
+evalModel (
-  { config, ... }:
+  { config, modulesPath, ... }:
   {
     imports = [
       ./resources
@@ -27,81 +44,107 @@ in
     ];
     options =
       {
-        operator = lib.mkOption {
-          default = pkgs.writeShellScriptBin "deploy-apps.sh" (
-            lib.concatStringsSep "\n" (
-              lib.mapAttrsToList (app: _: ''
-                echo 'DEPLOYING APP: ${app}'
-                ${config.${app}}
-              '') host-mapping
-            )
-          );
-        };
         # get a typed reference to the app deployments to expose their `run`
         tags = lib.mkOption {
-          type = types.attrsOf config.deployment-type;
+          type = types.attrsOf config.env-output;
-          default = lib.mapAttrs (
+          default = lib.genAttrs (nodes ++ [ "all" ]) (
-            app: _:
+            app:
             config.environments.${app}.deployment {
               # these are the values used in recursion, i.e. those for ssh-host.nix
               configuration = config."default-configuration";
-              deployment-name = "default";
+              deployment-name = [
+                "tags"
+                app
+              ];
             }
-          ) host-mapping;
+          );
+        };
+        operator = lib.mkOption {
+          type = types.path;
+          default = lib.getExe config.tags.all.deployments.ssh-hosts.run;
         };
       }
-      // lib.mapAttrs (
+      // lib.genAttrs nodes (
-        app: _:
+        app:
         lib.mkOption {
           type = types.path;
-          default = lib.getExe config.tags.${app}.ssh-host.run;
+          default = lib.getExe config.tags.${app}.deployments.ssh-host.run;
         }
-      ) host-mapping;
+      );
     config = lib.mkMerge [
       {
-        environments = lib.mapAttrs (app: host: {
+        environments =
-          resources =
+          {
-            {
+            "all" =
-              "external".garage = { };
+              { ... }:
-              "fediversity".nixos-module = { };
+              {
-              "age".secrets = { };
+                implementation =
-            }
+                  { ... }:
-            // lib.mapAttrs (_: host: {
+                  {
-              network = (import ../../machines/operator/${host}).fediversityVm;
+                    deployments.ssh-hosts = {
-            }) host-mapping;
+                      inherit
-          implementation =
+                        system
-            {
+                        root-path
-              deployment-name,
+                        caller
-              ...
+                        args
-            }:
+                        ;
-            {
+                      nodes = lib.lists.map (app: {
-              # try and use `ssh-host` since as of writing there is no plural variant
+                        # the separate invocations' `nixos-configuration` doubles to expose that both here and to their and this recursion
-              ssh-host = {
+                        inherit (config.tags.${app}.deployments.ssh-host) nixos-configuration ssh;
-                inherit system;
+                        deployment-name = [
-                ssh = {
+                          "tags"
-                  username = "root";
+                          app
-                  host = "${host}.abundos.eu";
+                        ];
-                  key-file = null;
+                      }) nodes;
-                  inherit sshOpts;
-                };
-                inherit deployment-name;
-                root-path = pathToRoot;
-                # recursion happens on the level of ssh-single-host, so let's go by that
-                caller = "deployment/fediversity/ssh-host.nix";
-                args = {
-                  inherit system;
-                  nodeName = "${host}.abundos.eu";
-                  network = (import ../../machines/operator/${host}).fediversityVm;
-                  conf = lib.recursiveUpdate conf {
-                    default-configuration.applications = lib.mapAttrs (_app: _: { enable = false; }) host-mapping // {
-                      ${app}.enable = true;
                     };
                   };
-                };
-                # omitting `nixos-configuration` as it's instead passed thru recursion's `ssh-host`
               };
-            };
+          }
-        }) host-mapping;
+          // lib.mapAttrs (app: host: environment: {
+            inherit resources;
+            implementation =
+              {
+                required-resources,
+                deployment-name,
+                ...
+              }:
+              let
+                relevant-resources = getSomeAttrs [ app ] required-resources;
+                garage = environment.config.resources."external".garage.process relevant-resources;
+              in
+              {
+                ancilliaryRequests.garage = garage.garageSide;
+                # try and use `ssh-host` since as of writing there is no plural variant
+                deployments.ssh-host = {
+                  inherit
+                    system
+                    root-path
+                    deployment-name
+                    caller
+                    args
+                    ;
+                  ssh = {
+                    inherit sshOpts username key-file;
+                    host = "${host}.abundos.eu";
+                  };
+                  nixos-configuration = {
+                    imports =
+                      [
+                        ../../infra/common/nixos
+                        "${sources.disko}/module.nix"
+                        "${modulesPath}/profiles/qemu-guest.nix"
+                        (environment.config.resources.${app}.network.process relevant-resources)
+                        (environment.config.resources."age".secrets.process relevant-resources)
+                      ]
+                      ++ (environment.config.resources."fediversity".nixos-module.process relevant-resources)
+                      ++ garage.applicationSide
+                      ++ (lib.optionals (app == "garage") (
+                        [ garage.mainConfig ] ++ lib.concatMap (app': config.tags.${app'}.ancilliaryRequests.garage) apps
+                      ));
+                  };
+                };
+              };
+          }) hosts;
       }
       conf
       # splice global config into apps using it

deployment/run/default.nix

@@ -17,6 +17,7 @@ let
     submodule
     ;
   inherit (pkgs.callPackage ../utils.nix { })
+    evalOption
     mapKeys
     withPackages
     withEnv
@@ -46,7 +47,7 @@ let
           system = "${system}";
           configuration = (import "${root-path}/${caller}" (builtins.fromJSON "${
             lib.replaceStrings [ "\"" ] [ "\\\"" ] (lib.strings.toJSON args)
-          }")).${deployment-name}.${deployment-type}.nixos-configuration;
+          }")).${lib.concatStringsSep "." deployment-name}.deployments.${deployment-type}.nixos-configuration;
         }
       ''
     );
@@ -63,6 +64,10 @@ let
       }
     ];
   };
+  system = mkOption {
+    description = "The architecture of the system to deploy to.";
+    type = types.str;
+  };
   nixos-configuration = mkOption {
     description = "A NixOS configuration.";
     type = types.unspecified;
@@ -73,14 +78,18 @@ let
     description = "The calling module to obtain the NixOS configuration from.";
     type = types.str;
   };
+  root-path = mkOption {
+    description = "The path to the root of the repository.";
+    type = types.path;
+  };
   args = mkOption {
     description = "The arguments with which to call the module to obtain the NixOS configuration.";
     type = types.attrs;
   };
   deployment-name = mkOption {
     description = "The name of the deployment for which to obtain the NixOS configuration.";
-    type = types.str;
+    type = types.listOf types.str;
-    default = "default";
+    default = [ "default" ];
   };
   httpBackend = mkOption {
     description = "environment variables to configure the TF HTTP back-end, see <https://developer.hashicorp.com/terraform/language/backend/http#configuration-variables>";
@@ -198,7 +207,7 @@ let
     };
   };
 in
-{
+rec {
   inherit nixos-configuration;
   ssh-host = mkOption {
     description = "A deployment by SSH to update a single existing NixOS host.";
@@ -206,21 +215,15 @@ in
       { config, ... }:
       {
         options = {
-          system = mkOption {
-            description = "The architecture of the system to deploy to.";
-            type = types.str;
-          };
           inherit
             caller
             args
             deployment-name
             nixos-configuration
+            root-path
+            system
             ;
           ssh = host-ssh;
-          root-path = mkOption {
-            description = "The path to the root of the repository.";
-            type = types.path;
-          };
           run = mkOption {
             type = types.package;
             default =
@@ -258,7 +261,7 @@ in
                   };
                 };
               in
-              pkgs.writers.writeBashBin "deploy-sh.sh"
+              pkgs.writers.writeBashBin "deploy-ssh-host.sh"
                 (withPackages [
                   pkgs.jq
                 ])
@@ -270,6 +273,71 @@ in
       }
     );
   };
+  ssh-hosts = mkOption {
+    description = "A deployment by SSH to update multiple existing NixOS hosts in order.";
+    type = submodule (
+      { config, ... }:
+      {
+        options = {
+          inherit
+            caller
+            args
+            root-path
+            system
+            ;
+          nodes = mkOption {
+            type = types.listOf (submodule {
+              options = {
+                inherit deployment-name nixos-configuration;
+                ssh = host-ssh;
+              };
+            });
+          };
+          run = mkOption {
+            type = types.package;
+            default =
+              let
+                inherit (config)
+                  system
+                  caller
+                  args
+                  root-path
+                  nodes
+                  ;
+              in
+              pkgs.writers.writeBashBin "deploy-ssh-hosts.sh" (
+                lib.concatStringsSep "\n" (
+                  lib.lists.map (
+                    {
+                      deployment-name,
+                      ssh,
+                      nixos-configuration,
+                    }:
+                    ''
+                      set -e
+                      printf "\nDEPLOYING ${lib.concatStringsSep "." deployment-name}\n\n"
+                      ${lib.getExe
+                        (evalOption ssh-host {
+                          inherit
+                            system
+                            root-path
+                            caller
+                            args
+                            deployment-name
+                            ssh
+                            nixos-configuration
+                            ;
+                        }).run
+                      }
+                    ''
+                  ) nodes
+                )
+              );
+          };
+        };
+      }
+    );
+  };
   nixops4 = mkOption {
     description = "A NixOps4 NixOS deployment. For an example, see https://github.com/nixops4/nixops4-nixos/blob/main/example/deployment.nix.";
     type = nixops4Deployment;
@@ -280,22 +348,16 @@ in
       { config, ... }:
       {
         options = {
-          system = mkOption {
-            description = "The architecture of the system to deploy to.";
-            type = types.str;
-          };
           inherit
             caller
             args
             deployment-name
             httpBackend
             nixos-configuration
+            root-path
+            system
             ;
           ssh = host-ssh;
-          root-path = mkOption {
-            description = "The path to the root of the repository.";
-            type = types.path;
-          };
           run = mkOption {
             type = types.package;
             default =
@@ -354,11 +416,7 @@ in
       { config, ... }:
       {
         options = {
-          system = mkOption {
+          inherit httpBackend nixos-configuration system;
-            description = "The architecture of the system to deploy to.";
-            type = types.str;
-          };
-          inherit httpBackend nixos-configuration;
           ssh = host-ssh;
           node-name = mkOption {
             description = "the name of the ProxmoX node to use.";
@@ -439,22 +497,16 @@ in
       { config, ... }:
       {
         options = {
-          system = mkOption {
-            description = "The architecture of the system to deploy to.";
-            type = types.str;
-          };
           inherit
             caller
             args
             deployment-name
             httpBackend
             nixos-configuration
+            root-path
+            system
             ;
           ssh = host-ssh;
-          root-path = mkOption {
-            description = "The path to the root of the repository.";
-            type = types.path;
-          };
           node-name = mkOption {
             description = "the name of the ProxmoX node to use.";
             type = types.str;

deployment/utils-test.nix

@@ -2,6 +2,7 @@ let
   inherit (import ../default.nix { }) pkgs;
   inherit (pkgs.callPackage ./utils.nix { })
     mapKeys
+    getSomeAttrs
     evalOption
     toBash
     withPackages
@@ -22,6 +23,14 @@ in
     };
   };
 
+  test-getSomeAttrs = {
+    expr = getSomeAttrs [ "a" "c" ] {
+      a = 0;
+      b = 1;
+    };
+    expected.a = 0;
+  };
+
   test-evalOption = {
     expr = evalOption (mkOption {
       type = submodule {

deployment/utils.nix

@@ -14,6 +14,10 @@ rec {
       }
     );
 
+  getSomeAttrs =
+    names: attrs:
+    lib.getAttrs (lib.concatMap (name: lib.optional (lib.hasAttr name attrs) name) names) attrs;
+
   evalModel =
     module:
     (lib.evalModules {

flake.nix

@@ -73,6 +73,7 @@
                 inherit
                   (import ./deployment/fediversity {
                     inherit system host-mapping;
+                    ancilliary.garage = "test01";
                     conf."default-configuration" = default-configuration // {
                       enable = true;
                       applications = lib.mapAttrs (_app: _: { enable = true; }) host-mapping;