From cda46b5f659d237657767f5b2b77a749396e8412 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Thu, 28 Aug 2025 21:06:19 +0200 Subject: [PATCH] add data model test for TF --- .forgejo/workflows/ci.yaml | 6 ++ deployment/check/common/data-model.nix | 22 +++++ deployment/check/data-model-tf/constants.nix | 8 ++ deployment/check/data-model-tf/default.nix | 21 +++++ deployment/check/data-model-tf/deploy.sh | 35 ++++++++ deployment/check/data-model-tf/main.tf | 44 ++++++++++ deployment/check/data-model-tf/nixos.nix | 13 +++ deployment/check/data-model-tf/nixosTest.nix | 91 ++++++++++++++++++++ deployment/check/data-model-tf/setup.nix | 19 ++++ deployment/check/data-model-tf/tf-env.nix | 31 +++++++ deployment/check/data-model-tf/tf.nix | 11 +++ deployment/check/data-model-tf/variables.tf | 23 +++++ deployment/data-model.nix | 9 ++ deployment/flake-part.nix | 5 ++ 14 files changed, 338 insertions(+) create mode 100644 deployment/check/data-model-tf/constants.nix create mode 100644 deployment/check/data-model-tf/default.nix create mode 100644 deployment/check/data-model-tf/deploy.sh create mode 100644 deployment/check/data-model-tf/main.tf create mode 100644 deployment/check/data-model-tf/nixos.nix create mode 100644 deployment/check/data-model-tf/nixosTest.nix create mode 100644 deployment/check/data-model-tf/setup.nix create mode 100644 deployment/check/data-model-tf/tf-env.nix create mode 100644 deployment/check/data-model-tf/tf.nix create mode 100644 deployment/check/data-model-tf/variables.tf diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 626c9d13..94f81623 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -69,6 +69,12 @@ jobs: - uses: actions/checkout@v4 - run: nix build .#checks.x86_64-linux.deployment-model-nixops4 -L + check-deployment-model-tf: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix build .#checks.x86_64-linux.deployment-model-tf -L + ## NOTE: NixOps4 does not provide a good “dry run” mode, so we instead check ## proxies for resources, namely whether their `.#vmOptions.` and ## `.#nixosConfigurations.` outputs evaluate and build correctly, and diff --git a/deployment/check/common/data-model.nix b/deployment/check/common/data-model.nix index c4922f72..8928e93b 100644 --- a/deployment/check/common/data-model.nix +++ b/deployment/check/common/data-model.nix @@ -159,6 +159,20 @@ let }; }; }; + single-nixos-vm-tf = environment: { + resources."operator-environment".login-shell.username = "operator"; + implementation = requests: { + input = requests; + output.tf-host = { + nixos-configuration = mkNixosConfiguration environment requests; + ssh = { + username = "root"; + host = nodeName; + key-file = null; + }; + }; + }; + }; }; }; options = { @@ -185,6 +199,14 @@ let type = env.resource-mapping.output-type; default = env.deployment config."example-configuration"; }; + "tf-deployment" = + let + env = config.environments."single-nixos-vm-tf"; + in + mkOption { + type = env.resource-mapping.output-type; + default = env.deployment config."example-configuration"; + }; }; } ); diff --git a/deployment/check/data-model-tf/constants.nix b/deployment/check/data-model-tf/constants.nix new file mode 100644 index 00000000..a3e4e33a --- /dev/null +++ b/deployment/check/data-model-tf/constants.nix @@ -0,0 +1,8 @@ +{ + targetMachines = [ + "target" + ]; + pathToRoot = ../../..; + pathFromRoot = ./.; + enableAcme = true; +} diff --git a/deployment/check/data-model-tf/default.nix b/deployment/check/data-model-tf/default.nix new file mode 100644 index 00000000..1815f19a --- /dev/null +++ b/deployment/check/data-model-tf/default.nix @@ -0,0 +1,21 @@ +{ + runNixOSTest, + inputs, + sources, +}: + +runNixOSTest { + imports = [ + ../../data-model.nix + ../../function.nix + ../common/nixosTest.nix + ./nixosTest.nix + ]; + _module.args = { inherit inputs sources; }; + inherit (import ./constants.nix) + targetMachines + pathToRoot + pathFromRoot + enableAcme + ; +} diff --git a/deployment/check/data-model-tf/deploy.sh b/deployment/check/data-model-tf/deploy.sh new file mode 100644 index 00000000..c6f6414f --- /dev/null +++ b/deployment/check/data-model-tf/deploy.sh @@ -0,0 +1,35 @@ +#! /usr/bin/env bash +set -xeuo pipefail +declare username host system config_nix config_tf + +# INSTANTIATE +command=(nix-instantiate --argstr system "$system" --argstr config_nix "$config_nix" --argstr config_tf "$config_tf" ./nixos.nix) +# instantiate the config in /nix/store +"${command[@]}" -A out_path + +# DEPLOY +sshOpts=( + -o BatchMode=yes + -o StrictHostKeyChecking=no + # TODO set key for production + # ${if key-file == null then "" else "-i ${key-file}"} + # NOTE the below options are for tests + -o ConnectTimeout=1 + -o ServerAliveInterval=1 +) +destination="$username@$host" +# get the realized derivation to deploy +outPath=$(nix-store --realize "$("${command[@]}" --show-trace --eval --strict --json | jq -r '.drv_path')") +# deploy the config by nix-copy-closure +NIX_SSHOPTS="${sshOpts[*]}" nix-copy-closure --to "$destination" "$outPath" --gzip --use-substitutes +# switch the remote host to the config +# NOTE checks here are for tests - in production time-outs could be a real thing, rather than indicator of success! +# shellcheck disable=SC2029 +output=$(ssh "${sshOpts[@]}" "$destination" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; nohup $outPath/bin/switch-to-configuration switch &" 2>&1) || echo "status code: $?" +echo "output: $output" +if [[ $output != *"Timeout, server $host not responding"* ]]; then + echo "non-timeout error: $output" + exit 1 +else + exit 0 +fi diff --git a/deployment/check/data-model-tf/main.tf b/deployment/check/data-model-tf/main.tf new file mode 100644 index 00000000..a10dc19f --- /dev/null +++ b/deployment/check/data-model-tf/main.tf @@ -0,0 +1,44 @@ +# hash of our code directory, used to trigger re-deploy +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ../../..)\\\"}\""] +} + +# TF resource to build and deploy NixOS instances. +resource "terraform_data" "nixos" { + + # trigger rebuild/deploy if (FIXME?) any potentially used config/code changed, + # preventing these (20+s, build being bottleneck) when nothing changed. + # terraform-nixos separates these to only deploy if instantiate changed, + # yet building even then - which may be not as bad using deploy on remote. + # having build/deploy one resource reflects wanting to prevent no-op rebuilds + # over preventing (with less false positives) no-op deployments, + # as i could not find a way to do prevent no-op rebuilds without merging them: + # - generic resources cannot have outputs, while we want info from the instantiation (unless built on host?). + # - `data` always runs, which is slow for deploy and especially build. + triggers_replace = [ + data.external.hash.result, + var.host, + var.config_nix, + var.config_tf, + ] + + provisioner "local-exec" { + # directory to run the script from. we use the TF project root dir, + # here as a path relative from where TF is run from, + # matching calling modules' expectations on config_nix locations. + # note that absolute paths can cause false positives in triggers, + # so are generally discouraged in TF. + working_dir = path.root + environment = { + system = var.system + username = var.username + host = var.host + config_nix = var.config_nix + config_tf = replace(jsonencode(var.config_tf), "\"", "\\\"") + } + # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs? + # TODO: build on target? + command = "sh deploy.sh" + } +} diff --git a/deployment/check/data-model-tf/nixos.nix b/deployment/check/data-model-tf/nixos.nix new file mode 100644 index 00000000..e5c2d3ed --- /dev/null +++ b/deployment/check/data-model-tf/nixos.nix @@ -0,0 +1,13 @@ +{ + system, + config_nix, + config_tf, +}: +import ../../nixos.nix { + inherit system; + configuration = + (import ../common/data-model.nix { + inherit system; + config = config_nix // builtins.fromJSON config_tf; + })."tf-deployment".tf-host.nixos-configuration; +} diff --git a/deployment/check/data-model-tf/nixosTest.nix b/deployment/check/data-model-tf/nixosTest.nix new file mode 100644 index 00000000..11ec23e2 --- /dev/null +++ b/deployment/check/data-model-tf/nixosTest.nix @@ -0,0 +1,91 @@ +{ + lib, + config, + pkgs, + inputs, + ... +}: +let + inherit (import ./constants.nix) pathToRoot pathFromRoot; + inherit (pkgs) system; + # escapedJson = v: lib.replaceStrings [ "\"" ] [ "\\\\\"" ] (lib.strings.toJSON v); + deployment-config = { + inherit pathToRoot pathFromRoot; + inherit (config) enableAcme; + acmeNodeIP = if config.enableAcme then config.nodes.acme.networking.primaryIPAddress else null; + nodeName = "target"; + }; + inherit + ((import ../common/data-model.nix { + inherit system inputs; + config = deployment-config; + })."tf-deployment".tf-host.ssh + ) + host + username + # key-file + ; + tf-vars = { + inherit host username system; + config_nix = lib.strings.toJSON deployment-config; + # config_nix = escapedJson deployment-config; + # config_tf = ; + }; + tf-env = pkgs.callPackage ./tf-env.nix { }; +in +{ + _class = "nixosTest"; + imports = [ + ../common/data-model-options.nix + ]; + + name = "deployment-model"; + sourceFileset = lib.fileset.unions [ + ../../data-model.nix + ../../function.nix + ../common/data-model.nix + ../common/data-model-options.nix + ./constants.nix + ./main.tf + ./variables.tf + ./deploy.sh + ]; + + nodes.deployer = + { pkgs, ... }: + { + # nixpkgs.config.allowUnfree = lib.mkForce true; + + environment.systemPackages = with pkgs; [ + (pkgs.callPackage ./tf.nix { }) + jq + ]; + + # needed only when building from deployer + system.extraDependenciesFromModule = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + hello + ]; + }; + }; + + extraTestScript = '' + with subtest("ssh: Check the status before deployment"): + target.fail("hello 1>&2") + + with subtest("ssh: Run the deployment"): + deployer.succeed(""" + set -xeuo pipefail + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (k: v: ''export TF_VAR_${k}='${v}';'') tf-vars)} + export TF_LOG=info + + cd "${tf-env}/deployment/check/data-model-tf" + # parallelism=1: limit OOM risk + tofu apply --auto-approve -lock=false -parallelism=1 + """) + target.wait_for_unit("multi-user.target") + target.succeed("su - operator -c hello 1>&2") + ''; +} diff --git a/deployment/check/data-model-tf/setup.nix b/deployment/check/data-model-tf/setup.nix new file mode 100644 index 00000000..6a9cefc0 --- /dev/null +++ b/deployment/check/data-model-tf/setup.nix @@ -0,0 +1,19 @@ +{ + pkgs, + lib, + sources, +}: +pkgs.writeScriptBin "setup" '' + # calculated pins + echo '${lib.strings.toJSON sources}' > ./.npins.json + # generate TF lock for nix's TF providers + for category in deployment/check/data-model-tf; do + pushd "$category" + rm -rf .terraform/ + rm -f .terraform.lock.hcl + # suppress warning on architecture-specific generated lock file: + # `Warning: Incomplete lock file information for providers`. + tofu init -input=false 1>/dev/null + popd + done +'' diff --git a/deployment/check/data-model-tf/tf-env.nix b/deployment/check/data-model-tf/tf-env.nix new file mode 100644 index 00000000..cb22201d --- /dev/null +++ b/deployment/check/data-model-tf/tf-env.nix @@ -0,0 +1,31 @@ +{ + lib, + pkgs, + sources ? import ../../../npins, +}: +pkgs.stdenv.mkDerivation { + name = "tf-repo"; + src = + with lib.fileset; + toSource { + root = ../../../.; + # don't copy ignored files + fileset = intersection (gitTracked ../../../.) ../../../.; + }; + buildInputs = [ + (pkgs.callPackage ./tf.nix { }) + (pkgs.callPackage ./setup.nix { inherit sources; }) + ]; + buildPhase = '' + runHook preBuild + pushd deployment/check/data-model-tf + setup + popd + runHook postBuild + ''; + installPhase = '' + runHook preInstall + cp -r . $out + runHook postInstall + ''; +} diff --git a/deployment/check/data-model-tf/tf.nix b/deployment/check/data-model-tf/tf.nix new file mode 100644 index 00000000..8551cb82 --- /dev/null +++ b/deployment/check/data-model-tf/tf.nix @@ -0,0 +1,11 @@ +# FIXME: use overlays so this gets imported just once? +{ + pkgs, + ... +}: +let + tf = pkgs.opentofu; +in +tf.withPlugins (p: [ + p.external +]) diff --git a/deployment/check/data-model-tf/variables.tf b/deployment/check/data-model-tf/variables.tf new file mode 100644 index 00000000..c1fd0bf9 --- /dev/null +++ b/deployment/check/data-model-tf/variables.tf @@ -0,0 +1,23 @@ +variable "system" { + type = string + default = "x86_64-linux" +} + +variable "username" { + type = string + default = "root" +} + +variable "host" { + type = string +} + +variable "config_nix" { + type = string + default = "{}" +} + +variable "config_tf" { + type = map(any) + default = {} +} diff --git a/deployment/data-model.nix b/deployment/data-model.nix index c6fcc631..914a5e0f 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -82,6 +82,15 @@ let description = "A NixOps4 NixOS deployment. For an example, see https://github.com/nixops4/nixops4-nixos/blob/main/example/deployment.nix."; type = nixops4Deployment; }; + tf-host = mkOption { + description = "A Terraform deployment by SSH to update a single existing NixOS host."; + type = submodule { + options = { + inherit nixos-configuration; + ssh = host-ssh; + }; + }; + }; }; in { diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix index 2d7c7e39..ece89360 100644 --- a/deployment/flake-part.nix +++ b/deployment/flake-part.nix @@ -31,6 +31,11 @@ inherit (pkgs.testers) runNixOSTest; inherit inputs sources; }; + + deployment-model-tf = import ./check/data-model-tf { + inherit (pkgs.testers) runNixOSTest; + inherit inputs sources; + }; }; }; }