reusable TF deployment

note that, other than being easier to call, this maintains the TF
deployment's status of remaining a glorified wrapper of the SSH
deployment.

deployment/check/common/data-model-options.nix

@@ -3,13 +3,23 @@
   ...
 }:
 let
-  inherit (lib) types;
+  inherit (lib) mkOption types;
 in
 {
   options = {
-    host = lib.mkOption {
+    host = mkOption {
       type = types.str;
       description = "name of the host to deploy to";
     };
+    targetSystem = mkOption {
+      type = types.str;
+      description = "name of the host to deploy to";
+    };
+    sshOpts = mkOption {
+      description = "Extra SSH options (`-o`) to use.";
+      type = types.listOf types.str;
+      default = [ ];
+      example = "ConnectTimeout=60";
+    };
   };
 }

deployment/check/common/data-model.nix

@@ -1,17 +1,29 @@
 {
   config,
   system,
-  inputs ? (import ../../../default.nix { }).inputs,
+  inputs ? (import ../../../default.nix { }).inputs, # XXX can't be serialized
   sources ? import ../../../npins,
   ...
-}:
+}@args:
 
 let
+  # having this module's location (`self`) and (serializable) `args`, we know
+  # enough to make it re-call itself to extract different info elsewhere later.
+  # we use this to make a deployment script using the desired nixos config,
+  # which would otherwise not be serializable, while nix also makes it hard to
+  # produce its derivation to pass thru without a `nix-instantiate` call,
+  # which in turn would need to be passed the (unserializable) nixos config.
+  self = "deployment/check/common/data-model.nix";
   inherit (sources) nixpkgs;
   pkgs = import nixpkgs { inherit system; };
   inherit (pkgs) lib;
   deployment-config = config;
-  inherit (deployment-config) nodeName;
+  inherit (deployment-config)
+    nodeName
+    pathToRoot
+    targetSystem
+    sshOpts
+    ;
   inherit (lib) mkOption types;
   eval =
     module:
@@ -124,17 +136,22 @@ let
           {
             single-nixos-vm-ssh = environment: {
               resources."operator-environment".login-shell.username = "operator";
-              implementation = requests: {
+              implementation =
-                input = requests;
+                {
-                output.ssh-host = {
+                  required-resources,
-                  nixos-configuration = mkNixosConfiguration environment requests;
+                  ...
-                  ssh = {
+                }:
-                    username = "root";
+                {
-                    host = nodeName;
+                  input = required-resources;
-                    key-file = null;
+                  output.ssh-host = {
+                    nixos-configuration = mkNixosConfiguration environment required-resources;
+                    ssh = {
+                      username = "root";
+                      host = nodeName;
+                      key-file = null;
+                    };
                   };
                 };
-              };
             };
             single-nixos-vm-nixops4 = environment: {
               resources."operator-environment".login-shell.username = "operator";
@@ -161,17 +178,27 @@ let
             };
             single-nixos-vm-tf = environment: {
               resources."operator-environment".login-shell.username = "operator";
-              implementation = requests: {
+              implementation =
-                input = requests;
+                {
-                output.tf-host = {
+                  required-resources,
-                  nixos-configuration = mkNixosConfiguration environment requests;
+                  deployment-name,
-                  ssh = {
+                }:
-                    username = "root";
+                {
-                    host = nodeName;
+                  input = required-resources;
-                    key-file = null;
+                  output.tf-host = {
+                    nixos-configuration = mkNixosConfiguration environment required-resources;
+                    system = targetSystem;
+                    ssh = {
+                      username = "root";
+                      host = nodeName;
+                      key-file = null;
+                      inherit sshOpts;
+                    };
+                    module = self;
+                    inherit args deployment-name;
+                    root-path = pathToRoot;
                   };
                 };
-              };
             };
           };
       };
@@ -189,7 +216,10 @@ let
           in
           mkOption {
             type = env.resource-mapping.output-type;
-            default = env.deployment config."example-configuration";
+            default = env.deployment {
+              deployment-name = "ssh-deployment";
+              configuration = config."example-configuration";
+            };
           };
         "nixops4-deployment" =
           let
@@ -197,7 +227,10 @@ let
           in
           mkOption {
             type = env.resource-mapping.output-type;
-            default = env.deployment config."example-configuration";
+            default = env.deployment {
+              deployment-name = "nixops4-deployment";
+              configuration = config."example-configuration";
+            };
           };
         "tf-deployment" =
           let
@@ -205,7 +238,10 @@ let
           in
           mkOption {
             type = env.resource-mapping.output-type;
-            default = env.deployment config."example-configuration";
+            default = env.deployment {
+              deployment-name = "tf-deployment";
+              configuration = config."example-configuration";
+            };
           };
       };
     }

deployment/check/data-model-tf/deploy.sh

@@ -1,35 +0,0 @@
-#! /usr/bin/env bash
-set -xeuo pipefail
-declare username host system config_nix config_tf
-
-# INSTANTIATE
-command=(nix-instantiate --argstr system "$system" --argstr config_nix "$config_nix" --argstr config_tf "$config_tf" ./nixos.nix)
-# instantiate the config in /nix/store
-"${command[@]}" -A out_path
-
-# DEPLOY
-sshOpts=(
-  -o BatchMode=yes
-  -o StrictHostKeyChecking=no
-  # TODO set key for production
-  # ${if key-file == null then "" else "-i ${key-file}"}
-  # NOTE the below options are for tests
-  -o ConnectTimeout=1
-  -o ServerAliveInterval=1
-)
-destination="$username@$host"
-# get the realized derivation to deploy
-outPath=$(nix-store --realize "$("${command[@]}" --show-trace --eval --strict --json | jq -r '.drv_path')")
-# deploy the config by nix-copy-closure
-NIX_SSHOPTS="${sshOpts[*]}" nix-copy-closure --to "$destination" "$outPath" --gzip --use-substitutes
-# switch the remote host to the config
-# NOTE checks here are for tests - in production time-outs could be a real thing, rather than indicator of success!
-# shellcheck disable=SC2029
-output=$(ssh "${sshOpts[@]}" "$destination" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; nohup $outPath/bin/switch-to-configuration switch &" 2>&1) || echo "status code: $?"
-echo "output: $output"
-if [[ $output != *"Timeout, server $host not responding"* ]]; then
-  echo "non-timeout error: $output"
-  exit 1
-else
-  exit 0
-fi

deployment/check/data-model-tf/nixos.nix

@@ -1,13 +0,0 @@
-{
-  system,
-  config_nix,
-  config_tf,
-}:
-import ../../nixos.nix {
-  inherit system;
-  configuration =
-    (import ../common/data-model.nix {
-      inherit system;
-      config = config_nix // builtins.fromJSON config_tf;
-    })."tf-deployment".tf-host.nixos-configuration;
-}

deployment/check/data-model-tf/nixosTest.nix

@@ -1,37 +1,26 @@
 {
   lib,
-  config,
   pkgs,
-  inputs,
   ...
 }:
 let
   inherit (import ./constants.nix) pathToRoot pathFromRoot;
   inherit (pkgs) system;
-  # escapedJson = v: lib.replaceStrings [ "\"" ] [ "\\\\\"" ] (lib.strings.toJSON v);
   deployment-config = {
     inherit pathToRoot pathFromRoot;
-    inherit (config) enableAcme;
-    acmeNodeIP = if config.enableAcme then config.nodes.acme.networking.primaryIPAddress else null;
     nodeName = "target";
+    targetSystem = system;
+    sshOpts = [
+      "ConnectTimeout=1"
+      "ServerAliveInterval=1"
+    ];
   };
-  inherit
+  deployment =
-    ((import ../common/data-model.nix {
+    (import ../common/data-model.nix {
-      inherit system inputs;
+      inherit system;
       config = deployment-config;
-    })."tf-deployment".tf-host.ssh
+      # opt not to pass `inputs`, as we could only pass serializable arguments through to its self-call
-    )
+    })."tf-deployment".tf-host;
-    host
-    username
-    # key-file
-    ;
-  tf-vars = {
-    inherit host username system;
-    config_nix = lib.strings.toJSON deployment-config;
-    # config_nix = escapedJson deployment-config;
-    # config_tf = ;
-  };
-  tf-env = pkgs.callPackage ./tf-env.nix { };
 in
 {
   _class = "nixosTest";
@@ -41,23 +30,14 @@ in
 
   name = "deployment-model";
   sourceFileset = lib.fileset.unions [
-    ../../data-model.nix
+    ../../run/tf-single-host/run.sh
-    ../../function.nix
-    ../common/data-model.nix
-    ../common/data-model-options.nix
-    ./constants.nix
-    ./main.tf
-    ./variables.tf
-    ./deploy.sh
   ];
 
   nodes.deployer =
     { pkgs, ... }:
     {
-      # nixpkgs.config.allowUnfree = lib.mkForce true;
-
       environment.systemPackages = with pkgs; [
-        (pkgs.callPackage ./tf.nix { })
+        (pkgs.callPackage ../../run/tf-single-host/tf.nix { })
         jq
       ];
 
@@ -77,13 +57,7 @@ in
 
     with subtest("ssh: Run the deployment"):
       deployer.succeed("""
-        set -xeuo pipefail
+        ${deployment.run}
-        ${lib.concatStringsSep "\n" (lib.mapAttrsToList (k: v: ''export TF_VAR_${k}='${v}';'') tf-vars)}
-        export TF_LOG=info
-
-        cd "${tf-env}/deployment/check/data-model-tf"
-        # parallelism=1: limit OOM risk
-        tofu apply --auto-approve -lock=false -parallelism=1
       """)
       target.wait_for_unit("multi-user.target")
       target.succeed("su - operator -c hello 1>&2")

deployment/check/data-model-tf/setup.nix

@@ -1,19 +0,0 @@
-{
-  pkgs,
-  lib,
-  sources,
-}:
-pkgs.writeScriptBin "setup" ''
-  # calculated pins
-  echo '${lib.strings.toJSON sources}' > ./.npins.json
-  # generate TF lock for nix's TF providers
-  for category in deployment/check/data-model-tf; do
-    pushd "$category"
-    rm -rf .terraform/
-    rm -f .terraform.lock.hcl
-    # suppress warning on architecture-specific generated lock file:
-    # `Warning: Incomplete lock file information for providers`.
-    tofu init -input=false 1>/dev/null
-    popd
-  done
-''

deployment/check/data-model-tf/variables.tf

@@ -1,23 +0,0 @@
-variable "system" {
-  type = string
-  default = "x86_64-linux"
-}
-
-variable "username" {
-  type = string
-  default = "root"
-}
-
-variable "host" {
-  type = string
-}
-
-variable "config_nix" {
-  type = string
-  default = "{}"
-}
-
-variable "config_tf" {
-  type = map(any)
-  default = {}
-}

deployment/data-model.nix

@@ -18,6 +18,16 @@ let
     str
     submodule
     ;
+  toBash =
+    v:
+    lib.replaceStrings [ "\"" ] [ "\\\\\"" ] (
+      if lib.isPath v || builtins.isNull v then
+        toString v
+      else if lib.isString v then
+        v
+      else
+        lib.strings.toJSON v
+    );
 
   functionType = import ./function.nix;
   application-resources = submodule {
@@ -65,6 +75,12 @@ let
           type = nullOr str;
           example = "/root/.ssh/id_ed25519";
         };
+        sshOpts = mkOption {
+          description = "Extra SSH options (`-o`) to use.";
+          type = types.listOf str;
+          default = [ ];
+          example = "ConnectTimeout=60";
+        };
       };
     };
   };
@@ -84,12 +100,73 @@ let
     };
     tf-host = mkOption {
       description = "A Terraform deployment by SSH to update a single existing NixOS host.";
-      type = submodule {
+      type = submodule (tf-host: {
         options = {
+          system = mkOption {
+            description = "The architecture of the system to deploy to.";
+            type = types.str;
+          };
           inherit nixos-configuration;
           ssh = host-ssh;
+          module = mkOption {
+            description = "The module to call to obtain the NixOS configuration from.";
+            type = types.str;
+          };
+          args = mkOption {
+            description = "The arguments with which to call the module to obtain the NixOS configuration.";
+            type = types.attrs;
+          };
+          deployment-name = mkOption {
+            description = "The name of the deployment for which to obtain the NixOS configuration.";
+            type = types.str;
+          };
+          root-path = mkOption {
+            description = "The path to the root of the repository.";
+            type = types.path;
+          };
+          run = mkOption {
+            type = types.str;
+            # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times.
+            # readOnly = true;
+            default =
+              let
+                inherit (tf-host.config)
+                  system
+                  ssh
+                  module
+                  args
+                  deployment-name
+                  root-path
+                  ;
+                inherit (ssh)
+                  host
+                  username
+                  key-file
+                  sshOpts
+                  ;
+                environment = {
+                  key_file = key-file;
+                  deployment_name = deployment-name;
+                  root_path = root-path;
+                  ssh_opts = sshOpts;
+                  inherit
+                    system
+                    host
+                    username
+                    module
+                    args
+                    ;
+                  deployment_type = "tf-host";
+                };
+                tf-env = pkgs.callPackage ./run/tf-single-host/tf-env.nix { };
+              in
+              ''
+                env ${toString (lib.mapAttrsToList (k: v: "TF_VAR_${k}=\"${toBash v}\"") environment)} \
+                tf_env=${tf-env} bash ./deployment/run/tf-single-host/run.sh
+              '';
+          };
         };
-      };
+      });
     };
   };
 in
@@ -201,7 +278,16 @@ in
               type = submodule functionType;
               readOnly = true;
               default = {
-                input-type = attrsOf application-resources;
+                input-type = submodule {
+                  options = {
+                    deployment-name = mkOption {
+                      type = types.str;
+                    };
+                    required-resources = mkOption {
+                      type = attrsOf application-resources;
+                    };
+                  };
+                };
                 output-type = deployment-type;
               };
             };
@@ -213,14 +299,17 @@ in
               type = functionTo (environment.config.resource-mapping.output-type);
               readOnly = true;
               default =
-                cfg:
+                {
+                  deployment-name,
+                  configuration,
+                }:
                 # TODO: check cfg.enable.true
                 let
                   required-resources = lib.mapAttrs (
                     name: application-settings: config.applications.${name}.resources application-settings
-                  ) cfg.applications;
+                  ) configuration.applications;
                 in
-                (environment.config.implementation required-resources).output;
+                (environment.config.implementation { inherit required-resources deployment-name; }).output;
 
             };
           };

deployment/run/ssh-single-host/run.sh

@@ -0,0 +1,49 @@
+#! /usr/bin/env bash
+set -xeuo pipefail
+declare username host system module args deployment_name deployment_type key_file root_path ssh_opts
+IFS=" " read -r -a ssh_opts <<< "$( (echo "$ssh_opts" | jq -r '@sh') | tr -d \'\")"
+
+# DEPLOY
+sshOpts=(
+  -o BatchMode=yes
+  -o StrictHostKeyChecking=no
+)
+for ssh_opt in "${ssh_opts[@]}"; do
+  sshOpts+=(
+    -o "$ssh_opt"
+  )
+done
+if [[ -n "$key_file" ]]; then
+  sshOpts+=(
+    -i "$key_file"
+  )
+fi
+destination="$username@$host"
+
+command=(nix-instantiate --show-trace --expr "
+  import $root_path/deployment/nixos.nix {
+    system = \"$system\";
+    configuration = (import \"$root_path/$module\" (builtins.fromJSON ''$args'')).$deployment_name.$deployment_type.nixos-configuration;
+  }
+")
+
+# INSTANTIATE
+# instantiate the config in /nix/store
+"${command[@]}" -A out_path
+
+# get the realized derivation to deploy
+outPath=$(nix-store --realize "$("${command[@]}" --show-trace --eval --strict --json | jq -r '.drv_path')")
+# deploy the config by nix-copy-closure
+NIX_SSHOPTS="${sshOpts[*]}" nix-copy-closure --to "$destination" "$outPath" --gzip --use-substitutes
+# switch the remote host to the config
+# shellcheck disable=SC2029
+ssh "${sshOpts[@]}" "$destination" "nix-env --profile /nix/var/nix/profiles/system --set $outPath"
+# shellcheck disable=SC2029
+output=$(ssh "${sshOpts[@]}" "$destination" "nohup $outPath/bin/switch-to-configuration switch &" 2>&1) || echo "status code: $?"
+echo "output: $output"
+if [[ $output != *"Timeout, server $host not responding"* ]]; then
+  echo "non-timeout error: $output"
+  exit 1
+else
+  exit 0
+fi

deployment/run/tf-single-host/main.tf

@@ -19,8 +19,10 @@ resource "terraform_data" "nixos" {
   triggers_replace = [
     data.external.hash.result,
     var.host,
-    var.config_nix,
+    var.module,
-    var.config_tf,
+    var.args,
+    var.root_path,
+    var.deployment_type,
   ]
 
   provisioner "local-exec" {
@@ -34,11 +36,17 @@ resource "terraform_data" "nixos" {
       system = var.system
       username = var.username
       host = var.host
-      config_nix = var.config_nix
+      module = var.module
-      config_tf = replace(jsonencode(var.config_tf), "\"", "\\\"")
+      host = var.host
+      args = var.args
+      key_file = var.key_file
+      deployment_name = var.deployment_name
+      root_path = var.root_path
+      ssh_opts = var.ssh_opts
+      deployment_type = var.deployment_type
     }
     # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs?
     # TODO: build on target?
-    command = "sh deploy.sh"
+    command = "sh ../ssh-single-host/run.sh"
   }
 }

deployment/run/tf-single-host/run.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env bash
+set -xeuo pipefail
+declare tf_env
+
+export TF_LOG=info
+
+cd "${tf_env}/deployment/run/tf-single-host"
+# parallelism=1: limit OOM risk
+tofu apply --auto-approve -lock=false -parallelism=1

deployment/run/tf-single-host/setup.nix

@@ -0,0 +1,16 @@
+{
+  pkgs,
+  lib,
+  sources,
+}:
+pkgs.writeScriptBin "setup" ''
+  set -xe
+  # calculated pins
+  echo '${lib.strings.toJSON sources}' > ./.npins.json
+  # generate TF lock for nix's TF providers
+  rm -rf .terraform/
+  rm -f .terraform.lock.hcl
+  # suppress warning on architecture-specific generated lock file:
+  # `Warning: Incomplete lock file information for providers`.
+  tofu init -input=false 1>/dev/null
+''

deployment/run/tf-single-host/tf-env.nix

@@ -18,8 +18,8 @@ pkgs.stdenv.mkDerivation {
   ];
   buildPhase = ''
     runHook preBuild
-    pushd deployment/check/data-model-tf
+    pushd deployment/run/tf-single-host
-    setup
+    source setup
     popd
     runHook postBuild
   '';

deployment/run/tf-single-host/variables.tf

@@ -0,0 +1,54 @@
+variable "system" {
+  description = "The architecture of the system to deploy to."
+  type = string
+  default = "x86_64-linux"
+}
+
+variable "username" {
+  description = "the SSH user to use"
+  type = string
+  default = "root"
+}
+
+variable "host" {
+  description = "the host to access by SSH"
+  type = string
+}
+
+variable "module" {
+  description = "The module to call to obtain the NixOS configuration from."
+  type = string
+}
+
+variable "args" {
+  description = "The arguments with which to call the module to obtain the NixOS configuration."
+  type = string
+  default = "{}"
+}
+
+variable "key_file" {
+  description = "path to the user's SSH private key"
+  type = string
+}
+
+variable "deployment_name" {
+  description = "The name of the deployment for which to obtain the NixOS configuration."
+  type = string
+}
+
+variable "root_path" {
+  description = "The path to the root of the repository."
+  type = string
+}
+
+variable "ssh_opts" {
+  description = "Extra SSH options (`-o`) to use."
+  type = string
+  default = "[]"
+}
+
+variable "deployment_type" {
+  description = "A `deployment-type` from the Fediversity data model, for grabbing the desired NixOS configuration."
+  type = string
+  default = "tf-host"
+}