From aefbf47a74be517b6152a34bd5d411a76a1d09c5 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 1 Sep 2025 13:30:58 +0200 Subject: [PATCH] reusable TF deployment note that, other than being easier to call, this maintains the TF deployment's status of remaining a glorified wrapper of the SSH deployment. --- .../check/common/data-model-options.nix | 14 ++- deployment/check/common/data-model.nix | 84 ++++++++++----- deployment/check/data-model-tf/deploy.sh | 35 ------ deployment/check/data-model-tf/nixos.nix | 13 --- deployment/check/data-model-tf/nixosTest.nix | 52 +++------ deployment/check/data-model-tf/setup.nix | 19 ---- deployment/check/data-model-tf/variables.tf | 23 ---- deployment/data-model.nix | 101 ++++++++++++++++-- deployment/run/ssh-single-host/run.sh | 49 +++++++++ .../tf-single-host}/main.tf | 18 +++- deployment/run/tf-single-host/run.sh | 9 ++ deployment/run/tf-single-host/setup.nix | 16 +++ .../tf-single-host}/tf-env.nix | 4 +- .../tf-single-host}/tf.nix | 0 deployment/run/tf-single-host/variables.tf | 54 ++++++++++ 15 files changed, 323 insertions(+), 168 deletions(-) delete mode 100644 deployment/check/data-model-tf/deploy.sh delete mode 100644 deployment/check/data-model-tf/nixos.nix delete mode 100644 deployment/check/data-model-tf/setup.nix delete mode 100644 deployment/check/data-model-tf/variables.tf create mode 100644 deployment/run/ssh-single-host/run.sh rename deployment/{check/data-model-tf => run/tf-single-host}/main.tf (82%) create mode 100644 deployment/run/tf-single-host/run.sh create mode 100644 deployment/run/tf-single-host/setup.nix rename deployment/{check/data-model-tf => run/tf-single-host}/tf-env.nix (90%) rename deployment/{check/data-model-tf => run/tf-single-host}/tf.nix (100%) create mode 100644 deployment/run/tf-single-host/variables.tf diff --git a/deployment/check/common/data-model-options.nix b/deployment/check/common/data-model-options.nix index 8492bee3..57dc88c5 100644 --- a/deployment/check/common/data-model-options.nix +++ b/deployment/check/common/data-model-options.nix @@ -3,13 +3,23 @@ ... }: let - inherit (lib) types; + inherit (lib) mkOption types; in { options = { - host = lib.mkOption { + host = mkOption { type = types.str; description = "name of the host to deploy to"; }; + targetSystem = mkOption { + type = types.str; + description = "name of the host to deploy to"; + }; + sshOpts = mkOption { + description = "Extra SSH options (`-o`) to use."; + type = types.listOf types.str; + default = [ ]; + example = "ConnectTimeout=60"; + }; }; } diff --git a/deployment/check/common/data-model.nix b/deployment/check/common/data-model.nix index 8928e93b..de292990 100644 --- a/deployment/check/common/data-model.nix +++ b/deployment/check/common/data-model.nix @@ -1,17 +1,29 @@ { config, system, - inputs ? (import ../../../default.nix { }).inputs, + inputs ? (import ../../../default.nix { }).inputs, # XXX can't be serialized sources ? import ../../../npins, ... -}: +}@args: let + # having this module's location (`self`) and (serializable) `args`, we know + # enough to make it re-call itself to extract different info elsewhere later. + # we use this to make a deployment script using the desired nixos config, + # which would otherwise not be serializable, while nix also makes it hard to + # produce its derivation to pass thru without a `nix-instantiate` call, + # which in turn would need to be passed the (unserializable) nixos config. + self = "deployment/check/common/data-model.nix"; inherit (sources) nixpkgs; pkgs = import nixpkgs { inherit system; }; inherit (pkgs) lib; deployment-config = config; - inherit (deployment-config) nodeName; + inherit (deployment-config) + nodeName + pathToRoot + targetSystem + sshOpts + ; inherit (lib) mkOption types; eval = module: @@ -124,17 +136,22 @@ let { single-nixos-vm-ssh = environment: { resources."operator-environment".login-shell.username = "operator"; - implementation = requests: { - input = requests; - output.ssh-host = { - nixos-configuration = mkNixosConfiguration environment requests; - ssh = { - username = "root"; - host = nodeName; - key-file = null; + implementation = + { + required-resources, + ... + }: + { + input = required-resources; + output.ssh-host = { + nixos-configuration = mkNixosConfiguration environment required-resources; + ssh = { + username = "root"; + host = nodeName; + key-file = null; + }; }; }; - }; }; single-nixos-vm-nixops4 = environment: { resources."operator-environment".login-shell.username = "operator"; @@ -161,17 +178,27 @@ let }; single-nixos-vm-tf = environment: { resources."operator-environment".login-shell.username = "operator"; - implementation = requests: { - input = requests; - output.tf-host = { - nixos-configuration = mkNixosConfiguration environment requests; - ssh = { - username = "root"; - host = nodeName; - key-file = null; + implementation = + { + required-resources, + deployment-name, + }: + { + input = required-resources; + output.tf-host = { + nixos-configuration = mkNixosConfiguration environment required-resources; + system = targetSystem; + ssh = { + username = "root"; + host = nodeName; + key-file = null; + inherit sshOpts; + }; + module = self; + inherit args deployment-name; + root-path = pathToRoot; }; }; - }; }; }; }; @@ -189,7 +216,10 @@ let in mkOption { type = env.resource-mapping.output-type; - default = env.deployment config."example-configuration"; + default = env.deployment { + deployment-name = "ssh-deployment"; + configuration = config."example-configuration"; + }; }; "nixops4-deployment" = let @@ -197,7 +227,10 @@ let in mkOption { type = env.resource-mapping.output-type; - default = env.deployment config."example-configuration"; + default = env.deployment { + deployment-name = "nixops4-deployment"; + configuration = config."example-configuration"; + }; }; "tf-deployment" = let @@ -205,7 +238,10 @@ let in mkOption { type = env.resource-mapping.output-type; - default = env.deployment config."example-configuration"; + default = env.deployment { + deployment-name = "tf-deployment"; + configuration = config."example-configuration"; + }; }; }; } diff --git a/deployment/check/data-model-tf/deploy.sh b/deployment/check/data-model-tf/deploy.sh deleted file mode 100644 index c6f6414f..00000000 --- a/deployment/check/data-model-tf/deploy.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /usr/bin/env bash -set -xeuo pipefail -declare username host system config_nix config_tf - -# INSTANTIATE -command=(nix-instantiate --argstr system "$system" --argstr config_nix "$config_nix" --argstr config_tf "$config_tf" ./nixos.nix) -# instantiate the config in /nix/store -"${command[@]}" -A out_path - -# DEPLOY -sshOpts=( - -o BatchMode=yes - -o StrictHostKeyChecking=no - # TODO set key for production - # ${if key-file == null then "" else "-i ${key-file}"} - # NOTE the below options are for tests - -o ConnectTimeout=1 - -o ServerAliveInterval=1 -) -destination="$username@$host" -# get the realized derivation to deploy -outPath=$(nix-store --realize "$("${command[@]}" --show-trace --eval --strict --json | jq -r '.drv_path')") -# deploy the config by nix-copy-closure -NIX_SSHOPTS="${sshOpts[*]}" nix-copy-closure --to "$destination" "$outPath" --gzip --use-substitutes -# switch the remote host to the config -# NOTE checks here are for tests - in production time-outs could be a real thing, rather than indicator of success! -# shellcheck disable=SC2029 -output=$(ssh "${sshOpts[@]}" "$destination" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; nohup $outPath/bin/switch-to-configuration switch &" 2>&1) || echo "status code: $?" -echo "output: $output" -if [[ $output != *"Timeout, server $host not responding"* ]]; then - echo "non-timeout error: $output" - exit 1 -else - exit 0 -fi diff --git a/deployment/check/data-model-tf/nixos.nix b/deployment/check/data-model-tf/nixos.nix deleted file mode 100644 index e5c2d3ed..00000000 --- a/deployment/check/data-model-tf/nixos.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ - system, - config_nix, - config_tf, -}: -import ../../nixos.nix { - inherit system; - configuration = - (import ../common/data-model.nix { - inherit system; - config = config_nix // builtins.fromJSON config_tf; - })."tf-deployment".tf-host.nixos-configuration; -} diff --git a/deployment/check/data-model-tf/nixosTest.nix b/deployment/check/data-model-tf/nixosTest.nix index 11ec23e2..0860457b 100644 --- a/deployment/check/data-model-tf/nixosTest.nix +++ b/deployment/check/data-model-tf/nixosTest.nix @@ -1,37 +1,26 @@ { lib, - config, pkgs, - inputs, ... }: let inherit (import ./constants.nix) pathToRoot pathFromRoot; inherit (pkgs) system; - # escapedJson = v: lib.replaceStrings [ "\"" ] [ "\\\\\"" ] (lib.strings.toJSON v); deployment-config = { inherit pathToRoot pathFromRoot; - inherit (config) enableAcme; - acmeNodeIP = if config.enableAcme then config.nodes.acme.networking.primaryIPAddress else null; nodeName = "target"; + targetSystem = system; + sshOpts = [ + "ConnectTimeout=1" + "ServerAliveInterval=1" + ]; }; - inherit - ((import ../common/data-model.nix { - inherit system inputs; + deployment = + (import ../common/data-model.nix { + inherit system; config = deployment-config; - })."tf-deployment".tf-host.ssh - ) - host - username - # key-file - ; - tf-vars = { - inherit host username system; - config_nix = lib.strings.toJSON deployment-config; - # config_nix = escapedJson deployment-config; - # config_tf = ; - }; - tf-env = pkgs.callPackage ./tf-env.nix { }; + # opt not to pass `inputs`, as we could only pass serializable arguments through to its self-call + })."tf-deployment".tf-host; in { _class = "nixosTest"; @@ -41,23 +30,14 @@ in name = "deployment-model"; sourceFileset = lib.fileset.unions [ - ../../data-model.nix - ../../function.nix - ../common/data-model.nix - ../common/data-model-options.nix - ./constants.nix - ./main.tf - ./variables.tf - ./deploy.sh + ../../run/tf-single-host/run.sh ]; nodes.deployer = { pkgs, ... }: { - # nixpkgs.config.allowUnfree = lib.mkForce true; - environment.systemPackages = with pkgs; [ - (pkgs.callPackage ./tf.nix { }) + (pkgs.callPackage ../../run/tf-single-host/tf.nix { }) jq ]; @@ -77,13 +57,7 @@ in with subtest("ssh: Run the deployment"): deployer.succeed(""" - set -xeuo pipefail - ${lib.concatStringsSep "\n" (lib.mapAttrsToList (k: v: ''export TF_VAR_${k}='${v}';'') tf-vars)} - export TF_LOG=info - - cd "${tf-env}/deployment/check/data-model-tf" - # parallelism=1: limit OOM risk - tofu apply --auto-approve -lock=false -parallelism=1 + ${deployment.run} """) target.wait_for_unit("multi-user.target") target.succeed("su - operator -c hello 1>&2") diff --git a/deployment/check/data-model-tf/setup.nix b/deployment/check/data-model-tf/setup.nix deleted file mode 100644 index 6a9cefc0..00000000 --- a/deployment/check/data-model-tf/setup.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - pkgs, - lib, - sources, -}: -pkgs.writeScriptBin "setup" '' - # calculated pins - echo '${lib.strings.toJSON sources}' > ./.npins.json - # generate TF lock for nix's TF providers - for category in deployment/check/data-model-tf; do - pushd "$category" - rm -rf .terraform/ - rm -f .terraform.lock.hcl - # suppress warning on architecture-specific generated lock file: - # `Warning: Incomplete lock file information for providers`. - tofu init -input=false 1>/dev/null - popd - done -'' diff --git a/deployment/check/data-model-tf/variables.tf b/deployment/check/data-model-tf/variables.tf deleted file mode 100644 index c1fd0bf9..00000000 --- a/deployment/check/data-model-tf/variables.tf +++ /dev/null @@ -1,23 +0,0 @@ -variable "system" { - type = string - default = "x86_64-linux" -} - -variable "username" { - type = string - default = "root" -} - -variable "host" { - type = string -} - -variable "config_nix" { - type = string - default = "{}" -} - -variable "config_tf" { - type = map(any) - default = {} -} diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 914a5e0f..4d4d0071 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -18,6 +18,16 @@ let str submodule ; + toBash = + v: + lib.replaceStrings [ "\"" ] [ "\\\\\"" ] ( + if lib.isPath v || builtins.isNull v then + toString v + else if lib.isString v then + v + else + lib.strings.toJSON v + ); functionType = import ./function.nix; application-resources = submodule { @@ -65,6 +75,12 @@ let type = nullOr str; example = "/root/.ssh/id_ed25519"; }; + sshOpts = mkOption { + description = "Extra SSH options (`-o`) to use."; + type = types.listOf str; + default = [ ]; + example = "ConnectTimeout=60"; + }; }; }; }; @@ -84,12 +100,73 @@ let }; tf-host = mkOption { description = "A Terraform deployment by SSH to update a single existing NixOS host."; - type = submodule { + type = submodule (tf-host: { options = { + system = mkOption { + description = "The architecture of the system to deploy to."; + type = types.str; + }; inherit nixos-configuration; ssh = host-ssh; + module = mkOption { + description = "The module to call to obtain the NixOS configuration from."; + type = types.str; + }; + args = mkOption { + description = "The arguments with which to call the module to obtain the NixOS configuration."; + type = types.attrs; + }; + deployment-name = mkOption { + description = "The name of the deployment for which to obtain the NixOS configuration."; + type = types.str; + }; + root-path = mkOption { + description = "The path to the root of the repository."; + type = types.path; + }; + run = mkOption { + type = types.str; + # error: The option `tf-deployment.tf-host.run' is read-only, but it's set multiple times. + # readOnly = true; + default = + let + inherit (tf-host.config) + system + ssh + module + args + deployment-name + root-path + ; + inherit (ssh) + host + username + key-file + sshOpts + ; + environment = { + key_file = key-file; + deployment_name = deployment-name; + root_path = root-path; + ssh_opts = sshOpts; + inherit + system + host + username + module + args + ; + deployment_type = "tf-host"; + }; + tf-env = pkgs.callPackage ./run/tf-single-host/tf-env.nix { }; + in + '' + env ${toString (lib.mapAttrsToList (k: v: "TF_VAR_${k}=\"${toBash v}\"") environment)} \ + tf_env=${tf-env} bash ./deployment/run/tf-single-host/run.sh + ''; + }; }; - }; + }); }; }; in @@ -201,7 +278,16 @@ in type = submodule functionType; readOnly = true; default = { - input-type = attrsOf application-resources; + input-type = submodule { + options = { + deployment-name = mkOption { + type = types.str; + }; + required-resources = mkOption { + type = attrsOf application-resources; + }; + }; + }; output-type = deployment-type; }; }; @@ -213,14 +299,17 @@ in type = functionTo (environment.config.resource-mapping.output-type); readOnly = true; default = - cfg: + { + deployment-name, + configuration, + }: # TODO: check cfg.enable.true let required-resources = lib.mapAttrs ( name: application-settings: config.applications.${name}.resources application-settings - ) cfg.applications; + ) configuration.applications; in - (environment.config.implementation required-resources).output; + (environment.config.implementation { inherit required-resources deployment-name; }).output; }; }; diff --git a/deployment/run/ssh-single-host/run.sh b/deployment/run/ssh-single-host/run.sh new file mode 100644 index 00000000..058f8267 --- /dev/null +++ b/deployment/run/ssh-single-host/run.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env bash +set -xeuo pipefail +declare username host system module args deployment_name deployment_type key_file root_path ssh_opts +IFS=" " read -r -a ssh_opts <<< "$( (echo "$ssh_opts" | jq -r '@sh') | tr -d \'\")" + +# DEPLOY +sshOpts=( + -o BatchMode=yes + -o StrictHostKeyChecking=no +) +for ssh_opt in "${ssh_opts[@]}"; do + sshOpts+=( + -o "$ssh_opt" + ) +done +if [[ -n "$key_file" ]]; then + sshOpts+=( + -i "$key_file" + ) +fi +destination="$username@$host" + +command=(nix-instantiate --show-trace --expr " + import $root_path/deployment/nixos.nix { + system = \"$system\"; + configuration = (import \"$root_path/$module\" (builtins.fromJSON ''$args'')).$deployment_name.$deployment_type.nixos-configuration; + } +") + +# INSTANTIATE +# instantiate the config in /nix/store +"${command[@]}" -A out_path + +# get the realized derivation to deploy +outPath=$(nix-store --realize "$("${command[@]}" --show-trace --eval --strict --json | jq -r '.drv_path')") +# deploy the config by nix-copy-closure +NIX_SSHOPTS="${sshOpts[*]}" nix-copy-closure --to "$destination" "$outPath" --gzip --use-substitutes +# switch the remote host to the config +# shellcheck disable=SC2029 +ssh "${sshOpts[@]}" "$destination" "nix-env --profile /nix/var/nix/profiles/system --set $outPath" +# shellcheck disable=SC2029 +output=$(ssh "${sshOpts[@]}" "$destination" "nohup $outPath/bin/switch-to-configuration switch &" 2>&1) || echo "status code: $?" +echo "output: $output" +if [[ $output != *"Timeout, server $host not responding"* ]]; then + echo "non-timeout error: $output" + exit 1 +else + exit 0 +fi diff --git a/deployment/check/data-model-tf/main.tf b/deployment/run/tf-single-host/main.tf similarity index 82% rename from deployment/check/data-model-tf/main.tf rename to deployment/run/tf-single-host/main.tf index a10dc19f..c05add7c 100644 --- a/deployment/check/data-model-tf/main.tf +++ b/deployment/run/tf-single-host/main.tf @@ -19,8 +19,10 @@ resource "terraform_data" "nixos" { triggers_replace = [ data.external.hash.result, var.host, - var.config_nix, - var.config_tf, + var.module, + var.args, + var.root_path, + var.deployment_type, ] provisioner "local-exec" { @@ -34,11 +36,17 @@ resource "terraform_data" "nixos" { system = var.system username = var.username host = var.host - config_nix = var.config_nix - config_tf = replace(jsonencode(var.config_tf), "\"", "\\\"") + module = var.module + host = var.host + args = var.args + key_file = var.key_file + deployment_name = var.deployment_name + root_path = var.root_path + ssh_opts = var.ssh_opts + deployment_type = var.deployment_type } # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs? # TODO: build on target? - command = "sh deploy.sh" + command = "sh ../ssh-single-host/run.sh" } } diff --git a/deployment/run/tf-single-host/run.sh b/deployment/run/tf-single-host/run.sh new file mode 100644 index 00000000..203466b9 --- /dev/null +++ b/deployment/run/tf-single-host/run.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env bash +set -xeuo pipefail +declare tf_env + +export TF_LOG=info + +cd "${tf_env}/deployment/run/tf-single-host" +# parallelism=1: limit OOM risk +tofu apply --auto-approve -lock=false -parallelism=1 diff --git a/deployment/run/tf-single-host/setup.nix b/deployment/run/tf-single-host/setup.nix new file mode 100644 index 00000000..4166812e --- /dev/null +++ b/deployment/run/tf-single-host/setup.nix @@ -0,0 +1,16 @@ +{ + pkgs, + lib, + sources, +}: +pkgs.writeScriptBin "setup" '' + set -xe + # calculated pins + echo '${lib.strings.toJSON sources}' > ./.npins.json + # generate TF lock for nix's TF providers + rm -rf .terraform/ + rm -f .terraform.lock.hcl + # suppress warning on architecture-specific generated lock file: + # `Warning: Incomplete lock file information for providers`. + tofu init -input=false 1>/dev/null +'' diff --git a/deployment/check/data-model-tf/tf-env.nix b/deployment/run/tf-single-host/tf-env.nix similarity index 90% rename from deployment/check/data-model-tf/tf-env.nix rename to deployment/run/tf-single-host/tf-env.nix index cb22201d..99d9e6c9 100644 --- a/deployment/check/data-model-tf/tf-env.nix +++ b/deployment/run/tf-single-host/tf-env.nix @@ -18,8 +18,8 @@ pkgs.stdenv.mkDerivation { ]; buildPhase = '' runHook preBuild - pushd deployment/check/data-model-tf - setup + pushd deployment/run/tf-single-host + source setup popd runHook postBuild ''; diff --git a/deployment/check/data-model-tf/tf.nix b/deployment/run/tf-single-host/tf.nix similarity index 100% rename from deployment/check/data-model-tf/tf.nix rename to deployment/run/tf-single-host/tf.nix diff --git a/deployment/run/tf-single-host/variables.tf b/deployment/run/tf-single-host/variables.tf new file mode 100644 index 00000000..32948210 --- /dev/null +++ b/deployment/run/tf-single-host/variables.tf @@ -0,0 +1,54 @@ +variable "system" { + description = "The architecture of the system to deploy to." + type = string + default = "x86_64-linux" +} + +variable "username" { + description = "the SSH user to use" + type = string + default = "root" +} + +variable "host" { + description = "the host to access by SSH" + type = string +} + +variable "module" { + description = "The module to call to obtain the NixOS configuration from." + type = string +} + +variable "args" { + description = "The arguments with which to call the module to obtain the NixOS configuration." + type = string + default = "{}" +} + +variable "key_file" { + description = "path to the user's SSH private key" + type = string +} + +variable "deployment_name" { + description = "The name of the deployment for which to obtain the NixOS configuration." + type = string +} + +variable "root_path" { + description = "The path to the root of the repository." + type = string +} + +variable "ssh_opts" { + description = "Extra SSH options (`-o`) to use." + type = string + default = "[]" +} + +variable "deployment_type" { + description = "A `deployment-type` from the Fediversity data model, for grabbing the desired NixOS configuration." + type = string + default = "tf-host" +}