wip: handle infra by TF

error example:

```
Error while installing hashicorp/external v2.3.4: the local package for
registry.opentofu.org/hashicorp/external 2.3.4 doesn't match any of the
checksums previously recorded in the dependency lock file (this might be
because the available checksums are for packages targeting different
platforms); for more information:
https://opentofu.org/docs/language/files/dependency-lock/#checksum-verification
```

.forgejo/workflows/ci.yaml

@@ -32,3 +32,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: cd launch && nix-build -A tests
+
+  check-infra:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: cd infra && nix-build -A tests

flake.nix

@@ -1,21 +1,14 @@
 {
-  inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
-    flake-parts.url = "github:hercules-ci/flake-parts";
-    git-hooks.url = "github:cachix/git-hooks.nix";
-    home-manager.url = "github:nix-community/home-manager";
-    home-manager.inputs.nixpkgs.follows = "nixpkgs";
-    agenix.url = "github:ryantm/agenix";
-
-    disko.url = "github:nix-community/disko";
-
-    nixops4.url = "github:nixops4/nixops4";
-    nixops4-nixos.url = "github:nixops4/nixops4-nixos";
-  };
-
   outputs =
-    inputs@{ flake-parts, ... }:
-    flake-parts.lib.mkFlake { inherit inputs; } {
+    { self, ... }:
+    let
+      sources = import ./npins;
+      inherit (sources) nixpkgs flake-parts git-hooks;
+      pkgs = import nixpkgs;
+      inherit (pkgs) lib;
+      flake-parts-lib = import "${flake-parts}/lib.nix" { inherit lib; };
+    in
+    flake-parts-lib.mkFlake { inherit self; } {
       systems = [
         "x86_64-linux"
         "aarch64-linux"
@@ -24,9 +17,7 @@
       ];
 
       imports = [
-        inputs.git-hooks.flakeModule
-        inputs.nixops4.modules.flake.default
-
+        (import "${git-hooks}/flake-module.nix")
         ./infra/flake-part.nix
         ./services/flake-part.nix
       ];
@@ -36,7 +27,6 @@
           config,
           pkgs,
           lib,
-          inputs',
           ...
         }:
         {
@@ -62,14 +52,10 @@
           devShells.default = pkgs.mkShell {
             packages = [
               pkgs.nil
-              inputs'.agenix.packages.default
+              (pkgs.callPackage "${agenix}/pkgs/agenix.nix" { })
               pkgs.openssh
               pkgs.httpie
               pkgs.jq
-              # exposing this env var as a hack to pass info in from form
-              (inputs'.nixops4.packages.default.overrideAttrs {
-                impureEnvVars = [ "DEPLOYMENT" ];
-              })
             ];
             shellHook = config.pre-commit.installationScript;
           };

infra/README.md

@@ -3,6 +3,28 @@
 This directory contains the definition of [the VMs](machines.md) that host our
 infrastructure.
 
+## requirements
+
+- [nix](https://nix.dev/)
+
+## usage
+
+### development
+
+before using other commands, if not using direnv:
+
+```sh
+nix-shell
+```
+
+then to initialize, or after updating pins or TF providers:
+
+```sh
+setup
+```
+
+then, one can use the `tofu` CLI.
+
 ## Provisioning VMs with an initial configuration
 
 NOTE[Niols]: This is very manual and clunky. Two things will happen. In the near

infra/TODO.md

@@ -0,0 +1,22 @@
+# differences
+
+differences between TF modules among JIT services (`launch/`) vs infra:
+
+- TF input variables (initialUser vs [host]domain) [including in triggers]
+- for_each (objects containing machines and their stuff)
+- nix modules
+- nix options
+- nix config
+- nix config passed in as TF
+- own dir with:
+  - TF config
+  - TF state
+  - TF lock
+  - `setup` process (document running per project)
+
+# todo
+
+what should be done to consolidate these:
+
+- abstract out common TF logic to a separate TF module
+- thru nix add as custom provider

infra/default.nix

@@ -0,0 +1,33 @@
+{
+  system ? builtins.currentSystem,
+  sources ? import ../npins,
+  pkgs ? import sources.nixpkgs { inherit system; },
+}:
+let
+  inherit (pkgs) lib;
+  setup = pkgs.writeScriptBin "setup" ''
+    echo '${lib.strings.toJSON sources}' > .npins.json
+    rm -rf .terraform/
+    tofu init
+  '';
+in
+{
+  # shell for testing TF directly
+  shell = pkgs.mkShellNoCC {
+    packages = [
+      (import ./../launch/tf.nix { inherit lib pkgs; })
+      pkgs.jaq
+      setup
+    ];
+  };
+
+  tests = pkgs.callPackage ./tests.nix { };
+
+  # re-export inputs so they can be overridden granularly
+  # (they can't be accessed from the outside any other way)
+  inherit
+    sources
+    system
+    pkgs
+    ;
+}

infra/machines/fedi200/dns.nix

@@ -0,0 +1,2 @@
+_: {
+}

infra/main.tf

@@ -0,0 +1,123 @@
+locals {
+  system = "x86_64-linux"
+  # dependency paths pre-calculated from npins
+  pins = jsondecode(file("${path.root}/.npins.json"))
+  # nix path: expose pins, use nixpkgs in flake commands (`nix run`)
+  nix_path = "${join(":", [for name, path in local.pins : "${name}=${path}"])}:flake=${local.pins["nixpkgs"]}:flake"
+}
+
+# hash of our code directory, used to trigger re-deploy
+# FIXME calculate separately to reduce false positives
+data "external" "hash" {
+  program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ..)\\\"}\""]
+}
+
+# TF resource to build and deploy NixOS instances.
+resource "terraform_data" "nixos" {
+
+  for_each = {
+    dns = "fedi200"
+    demo = "fedi201"
+    wiki = "vm02187"
+    forgejo = "vm02116"
+  }
+
+  # trigger rebuild/deploy if (FIXME?) any potentially used config/code changed,
+  # preventing these (20+s, build being bottleneck) when nothing changed.
+  # terraform-nixos separates these to only deploy if instantiate changed,
+  # yet building even then - which may be not as bad using deploy on remote.
+  # having build/deploy one resource reflects wanting to prevent no-op rebuilds
+  # over preventing (with less false positives) no-op deployments,
+  # as i could not find a way to do prevent no-op rebuilds without merging them:
+  # - generic resources cannot have outputs, while we want info from the instantiation (unless built on host?).
+  # - `data` always runs, which is slow for deploy and especially build.
+  triggers_replace = [
+    data.external.hash.result,
+    var.domain,
+    local.system,
+    each.key,
+    each.value,
+  ]
+
+  provisioner "local-exec" {
+    # directory to run the script from. we use the TF project root dir,
+    # here as a path relative from where TF is run from.
+    # note that absolute paths can cause false positives in triggers,
+    # so are generally discouraged in TF.
+    working_dir = path.root
+    environment = {
+      # nix path used on build, lets us refer to e.g. nixpkgs like `<nixpkgs>`
+      NIX_PATH = local.nix_path
+    }
+    # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs?
+    # TODO: build on target?
+    command = <<-EOF
+      set -euo pipefail
+
+      # INSTANTIATE
+      command=(
+        nix-instantiate
+        --expr
+        'let
+          os = import <nixpkgs/nixos> {
+            system = "${local.system}";
+            configuration = {
+              # note interpolations here TF ones
+              imports = [
+                # shared NixOS config
+                ${path.root}/../launch/shared.nix
+                # FIXME: separate template options by service
+                ${path.root}/options.nix
+                # FIXME: get VM details from TF
+                ${path.root}/machines/${each.value}
+                # for service `forgejo` import `forgejo.nix`
+                ${path.root}/machines/${each.value}/${each.key}.nix
+              ];
+              # nix path for debugging
+              nix.nixPath = [ "${local.nix_path}" ];
+            } //
+            # template parameters passed in from TF thru json
+            builtins.fromJSON "${replace(jsonencode({
+              terraform = {
+                domain = var.domain
+                hostname = each.value
+              }
+            }), "\"", "\\\"")}";
+          };
+        in
+        # info we want to get back out
+        {
+          substituters = builtins.concatStringsSep " " os.config.nix.settings.substituters;
+          trusted_public_keys = builtins.concatStringsSep " " os.config.nix.settings.trusted-public-keys;
+          drv_path = os.config.system.build.toplevel.drvPath;
+          out_path = os.config.system.build.toplevel;
+        }'
+      )
+      # instantiate the config in /nix/store
+      "$${command[@]}" -A out_path
+      # get the other info
+      json="$("$${command[@]}" --eval --strict --json)"
+
+      # DEPLOY
+      declare substituters trusted_public_keys drv_path
+      # set our variables using the json object
+      eval "export $(echo $json | jaq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"
+      host="root@${each.value}.${var.domain}" # FIXME: #24
+      buildArgs=(
+        --option extra-binary-caches https://cache.nixos.org/
+        --option substituters $substituters
+        --option trusted-public-keys $trusted_public_keys
+      )
+      sshOpts=(
+        -o BatchMode=yes
+        -o StrictHostKeyChecking=no
+      )
+      # get the realized derivation to deploy
+      outPath=$(nix-store --realize "$drv_path" "$${buildArgs[@]}")
+      # deploy the config by nix-copy-closure
+      NIX_SSHOPTS="$${sshOpts[*]}" nix-copy-closure --to "$host" "$outPath" --gzip --use-substitutes
+      # switch the remote host to the config
+      ssh "$${sshOpts[@]}" "$host" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; $outPath/bin/switch-to-configuration switch"
+    EOF
+  }
+}

infra/options.nix

@@ -0,0 +1,28 @@
+# TODO: could (part of) this be generated somehow? c.f #275
+{
+  lib,
+  ...
+}:
+let
+  inherit (lib) types mkOption;
+  inherit (types) str enum;
+in
+{
+  options.terraform = {
+    domain = mkOption {
+      type = enum [
+        "fediversity.net"
+      ];
+      description = ''
+        Apex domain under which the services will be deployed.
+      '';
+      default = "fediversity.net";
+    };
+    hostname = mkOption {
+      type = str;
+      description = ''
+        Internal name of the host, e.g. test01
+      '';
+    };
+  };
+}

infra/tests.nix

@@ -0,0 +1,29 @@
+{ lib, pkgs }:
+let
+  defaults = {
+    virtualisation = {
+      memorySize = 2048;
+      cores = 2;
+    };
+  };
+  tf = pkgs.callPackage ./../launch/tf.nix {
+    inherit lib pkgs;
+    dir = "infra/";
+  };
+  tfEnv = pkgs.callPackage ./../launch/tf-env.nix { };
+in
+lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) {
+  tf-validate = {
+    inherit defaults;
+    nodes.server = {
+      environment.systemPackages = [
+        tf
+        tfEnv
+      ];
+    };
+    testScript = ''
+      server.wait_for_unit("multi-user.target")
+      server.succeed("${lib.getExe tf} -chdir='${tfEnv}/infra' validate")
+    '';
+  };
+}

infra/variables.tf

@@ -0,0 +1,4 @@
+variable "domain" {
+  type = string
+  default = "abundos.eu"
+}

launch/default.nix

@@ -1,19 +1,13 @@
 {
   system ? builtins.currentSystem,
   sources ? import ../npins,
-  # match the same versions we deploy locally
-  inputs ? import sources.flake-inputs {
-    root = ../.;
-  },
-  # match the same version of opentofu that is deployed by the root flake
-  pkgs ? import inputs.nixpkgs {
-    inherit system;
-  },
+  pkgs ? import sources.nixpkgs { inherit system; },
 }:
 let
   inherit (pkgs) lib;
   setup = pkgs.writeScriptBin "setup" ''
     echo '${lib.strings.toJSON sources}' > .npins.json
+    rm -f .terraform.lock.hcl
     rm -rf .terraform/
     tofu init
   '';

launch/resource.nix

@@ -5,7 +5,7 @@
 }:
 
 let
-  inherit (lib) attrValues elem mkDefault;
+  inherit (lib) elem mkDefault;
   inherit (lib.attrsets) concatMapAttrs optionalAttrs;
   inherit (lib.strings) removeSuffix;
 
@@ -34,10 +34,4 @@ in
       ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}";
     }
   ) secrets;
-
-  ## FIXME: switch root authentication to users with password-less sudo, see #24
-  users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors ++ [
-    # allow our panel vm access to the test machines
-    keys.panel
-  ];
 }

launch/tests.nix

@@ -7,7 +7,10 @@ let
     };
   };
   tf = pkgs.callPackage ./tf.nix { };
-  tfEnv = pkgs.callPackage ./tf-env.nix { };
+  tfEnv = pkgs.callPackage ./tf-env.nix {
+    inherit lib pkgs;
+    dir = "launch/";
+  };
 in
 lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) {
   tf-validate = {

launch/tf-env.nix

@@ -1,6 +1,7 @@
 {
   lib,
   pkgs,
+  path,
   sources ? import ../npins,
   ...
 }:
@@ -18,7 +19,7 @@ pkgs.stdenv.mkDerivation {
   ];
   buildPhase = ''
     runHook preBuild
-    pushd launch/
+    pushd ${path}
     # calculated pins
     echo '${lib.strings.toJSON sources}' > .npins.json
     # generate TF lock for nix's TF providers

npins/sources.json

@@ -42,6 +42,32 @@
       "url": "https://github.com/fricklerhandwerk/flake-inputs/archive/559574c9cbb8af262f3944b67d60fbf0f6ad03c3.tar.gz",
       "hash": "0gbhmp6x2vdzvfnsvqzal3g8f8hx2ia6r73aibc78kazf78m67x6"
     },
+    "flake-parts": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "hercules-ci",
+        "repo": "flake-parts"
+      },
+      "branch": "main",
+      "submodules": false,
+      "revision": "c621e8422220273271f52058f618c94e405bb0f5",
+      "url": "https://github.com/hercules-ci/flake-parts/archive/c621e8422220273271f52058f618c94e405bb0f5.tar.gz",
+      "hash": "09j2dafd75ydlcw8v48vcpfm2mw0j6cs8286x2hha2lr08d232w4"
+    },
+    "git-hooks": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "cachix",
+        "repo": "git-hooks.nix"
+      },
+      "branch": "master",
+      "submodules": false,
+      "revision": "dcf5072734cb576d2b0c59b2ac44f5050b5eac82",
+      "url": "https://github.com/cachix/git-hooks.nix/archive/dcf5072734cb576d2b0c59b2ac44f5050b5eac82.tar.gz",
+      "hash": "1jmdxmx29xghjiaks6f5amnxld8w3kmxb2zv8lk2yzpgp6kr60qg"
+    },
     "htmx": {
       "type": "GitRelease",
       "repository": {

panel/nix/configuration.nix

@@ -29,7 +29,10 @@ let
       ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
       (builtins.toFile "extra-settings.py" cfg.extra-settings)
     ];
-    REPO_DIR = import ../../launch/tf-env.nix { inherit lib pkgs; };
+    REPO_DIR = import ../../launch/tf-env.nix {
+      inherit lib pkgs;
+      dir = "launch/";
+    };
   };
 
   python-environment = pkgs.python3.withPackages (

panel/nix/package.nix

@@ -60,7 +60,12 @@ python3.pkgs.buildPythonPackage {
     cp -v ${src}/manage.py $out/bin/manage.py
     chmod +x $out/bin/manage.py
     wrapProgram $out/bin/manage.py \
-      --set REPO_DIR "${import ../../launch/tf-env.nix { inherit lib pkgs; }}" \
+      --set REPO_DIR "${
+        import ../../launch/tf-env.nix {
+          inherit lib pkgs;
+          dir = "launch/";
+        }
+      }" \
       --prefix PYTHONPATH : "$PYTHONPATH"
     cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/
   '';