diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 5faf7cf4..48ce6bdc 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -32,3 +32,9 @@ jobs: steps: - uses: actions/checkout@v4 - run: cd launch && nix-build -A tests + + check-infra: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: cd infra && nix-build -A tests diff --git a/infra/README.md b/infra/README.md index 5fd01a90..4649ce54 100644 --- a/infra/README.md +++ b/infra/README.md @@ -3,6 +3,28 @@ This directory contains the definition of [the VMs](machines.md) that host our infrastructure. +## requirements + +- [nix](https://nix.dev/) + +## usage + +### development + +before using other commands, if not using direnv: + +```sh +nix-shell +``` + +then to initialize, or after updating pins or TF providers: + +```sh +setup +``` + +then, one can use the `tofu` CLI. + ## Provisioning VMs with an initial configuration NOTE[Niols]: This is very manual and clunky. Two things will happen. In the near diff --git a/infra/TODO.md b/infra/TODO.md new file mode 100644 index 00000000..ba1bc104 --- /dev/null +++ b/infra/TODO.md @@ -0,0 +1,22 @@ +# differences + +differences between TF modules among JIT services (`launch/`) vs infra: + +- TF input variables (initialUser vs [host]domain) [including in triggers] +- for_each (objects containing machines and their stuff) +- nix modules +- nix options +- nix config +- nix config passed in as TF +- own dir with: + - TF config + - TF state + - TF lock + - `setup` process (document running per project) + +# todo + +what should be done to consolidate these: + +- abstract out common TF logic to a separate TF module +- thru nix add as custom provider diff --git a/infra/default.nix b/infra/default.nix new file mode 100644 index 00000000..3e4b8bbd --- /dev/null +++ b/infra/default.nix @@ -0,0 +1,33 @@ +{ + system ? builtins.currentSystem, + sources ? import ../npins, + pkgs ? import sources.nixpkgs { inherit system; }, +}: +let + inherit (pkgs) lib; + setup = pkgs.writeScriptBin "setup" '' + echo '${lib.strings.toJSON sources}' > .npins.json + rm -rf .terraform/ + tofu init + ''; +in +{ + # shell for testing TF directly + shell = pkgs.mkShellNoCC { + packages = [ + (import ./../launch/tf.nix { inherit lib pkgs; }) + pkgs.jaq + setup + ]; + }; + + tests = pkgs.callPackage ./tests.nix { }; + + # re-export inputs so they can be overridden granularly + # (they can't be accessed from the outside any other way) + inherit + sources + system + pkgs + ; +} diff --git a/infra/machines/fedi200/dns.nix b/infra/machines/fedi200/dns.nix new file mode 100644 index 00000000..87a13d7f --- /dev/null +++ b/infra/machines/fedi200/dns.nix @@ -0,0 +1,2 @@ +_: { +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 00000000..9e97c8a8 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,123 @@ +locals { + system = "x86_64-linux" + # dependency paths pre-calculated from npins + pins = jsondecode(file("${path.root}/.npins.json")) + # nix path: expose pins, use nixpkgs in flake commands (`nix run`) + nix_path = "${join(":", [for name, path in local.pins : "${name}=${path}"])}:flake=${local.pins["nixpkgs"]}:flake" +} + +# hash of our code directory, used to trigger re-deploy +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ..)\\\"}\""] +} + +# TF resource to build and deploy NixOS instances. +resource "terraform_data" "nixos" { + + for_each = { + dns = "fedi200" + demo = "fedi201" + wiki = "vm02187" + forgejo = "vm02116" + } + + # trigger rebuild/deploy if (FIXME?) any potentially used config/code changed, + # preventing these (20+s, build being bottleneck) when nothing changed. + # terraform-nixos separates these to only deploy if instantiate changed, + # yet building even then - which may be not as bad using deploy on remote. + # having build/deploy one resource reflects wanting to prevent no-op rebuilds + # over preventing (with less false positives) no-op deployments, + # as i could not find a way to do prevent no-op rebuilds without merging them: + # - generic resources cannot have outputs, while we want info from the instantiation (unless built on host?). + # - `data` always runs, which is slow for deploy and especially build. + triggers_replace = [ + data.external.hash.result, + var.domain, + local.system, + each.key, + each.value, + ] + + provisioner "local-exec" { + # directory to run the script from. we use the TF project root dir, + # here as a path relative from where TF is run from. + # note that absolute paths can cause false positives in triggers, + # so are generally discouraged in TF. + working_dir = path.root + environment = { + # nix path used on build, lets us refer to e.g. nixpkgs like `` + NIX_PATH = local.nix_path + } + # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs? + # TODO: build on target? + command = <<-EOF + set -euo pipefail + + # INSTANTIATE + command=( + nix-instantiate + --expr + 'let + os = import { + system = "${local.system}"; + configuration = { + # note interpolations here TF ones + imports = [ + # shared NixOS config + ${path.root}/../launch/shared.nix + # FIXME: separate template options by service + ${path.root}/options.nix + # FIXME: get VM details from TF + ${path.root}/machines/${each.value} + # for service `forgejo` import `forgejo.nix` + ${path.root}/machines/${each.value}/${each.key}.nix + ]; + # nix path for debugging + nix.nixPath = [ "${local.nix_path}" ]; + } // + # template parameters passed in from TF thru json + builtins.fromJSON "${replace(jsonencode({ + terraform = { + domain = var.domain + hostname = each.value + } + }), "\"", "\\\"")}"; + }; + in + # info we want to get back out + { + substituters = builtins.concatStringsSep " " os.config.nix.settings.substituters; + trusted_public_keys = builtins.concatStringsSep " " os.config.nix.settings.trusted-public-keys; + drv_path = os.config.system.build.toplevel.drvPath; + out_path = os.config.system.build.toplevel; + }' + ) + # instantiate the config in /nix/store + "$${command[@]}" -A out_path + # get the other info + json="$("$${command[@]}" --eval --strict --json)" + + # DEPLOY + declare substituters trusted_public_keys drv_path + # set our variables using the json object + eval "export $(echo $json | jaq -r 'to_entries | map("\(.key)=\(.value)") | @sh')" + host="root@${each.value}.${var.domain}" # FIXME: #24 + buildArgs=( + --option extra-binary-caches https://cache.nixos.org/ + --option substituters $substituters + --option trusted-public-keys $trusted_public_keys + ) + sshOpts=( + -o BatchMode=yes + -o StrictHostKeyChecking=no + ) + # get the realized derivation to deploy + outPath=$(nix-store --realize "$drv_path" "$${buildArgs[@]}") + # deploy the config by nix-copy-closure + NIX_SSHOPTS="$${sshOpts[*]}" nix-copy-closure --to "$host" "$outPath" --gzip --use-substitutes + # switch the remote host to the config + ssh "$${sshOpts[@]}" "$host" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; $outPath/bin/switch-to-configuration switch" + EOF + } +} diff --git a/infra/options.nix b/infra/options.nix new file mode 100644 index 00000000..179e6add --- /dev/null +++ b/infra/options.nix @@ -0,0 +1,28 @@ +# TODO: could (part of) this be generated somehow? c.f #275 +{ + lib, + ... +}: +let + inherit (lib) types mkOption; + inherit (types) str enum; +in +{ + options.terraform = { + domain = mkOption { + type = enum [ + "fediversity.net" + ]; + description = '' + Apex domain under which the services will be deployed. + ''; + default = "fediversity.net"; + }; + hostname = mkOption { + type = str; + description = '' + Internal name of the host, e.g. test01 + ''; + }; + }; +} diff --git a/launch/garage.nix b/infra/test-machines/test01/garage.nix similarity index 100% rename from launch/garage.nix rename to infra/test-machines/test01/garage.nix diff --git a/launch/pixelfed.nix b/infra/test-machines/test04/pixelfed.nix similarity index 100% rename from launch/pixelfed.nix rename to infra/test-machines/test04/pixelfed.nix diff --git a/launch/peertube.nix b/infra/test-machines/test05/peertube.nix similarity index 100% rename from launch/peertube.nix rename to infra/test-machines/test05/peertube.nix diff --git a/launch/mastodon.nix b/infra/test-machines/test06/mastodon.nix similarity index 100% rename from launch/mastodon.nix rename to infra/test-machines/test06/mastodon.nix diff --git a/infra/tests.nix b/infra/tests.nix new file mode 100644 index 00000000..f2a3088b --- /dev/null +++ b/infra/tests.nix @@ -0,0 +1,29 @@ +{ lib, pkgs }: +let + defaults = { + virtualisation = { + memorySize = 2048; + cores = 2; + }; + }; + tf = pkgs.callPackage ./../launch/tf.nix { + inherit lib pkgs; + dir = "infra/"; + }; + tfEnv = pkgs.callPackage ./../launch/tf-env.nix { }; +in +lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) { + tf-validate = { + inherit defaults; + nodes.server = { + environment.systemPackages = [ + tf + tfEnv + ]; + }; + testScript = '' + server.wait_for_unit("multi-user.target") + server.succeed("${lib.getExe tf} -chdir='${tfEnv}/infra' validate") + ''; + }; +} diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 00000000..fe9da2ad --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,4 @@ +variable "domain" { + type = string + default = "abundos.eu" +} diff --git a/launch/resource.nix b/launch/resource.nix index ec719e20..4300005f 100644 --- a/launch/resource.nix +++ b/launch/resource.nix @@ -5,7 +5,7 @@ }: let - inherit (lib) attrValues elem mkDefault; + inherit (lib) elem mkDefault; inherit (lib.attrsets) concatMapAttrs optionalAttrs; inherit (lib.strings) removeSuffix; @@ -34,10 +34,4 @@ in ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}"; } ) secrets; - - ## FIXME: switch root authentication to users with password-less sudo, see #24 - users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors ++ [ - # allow our panel vm access to the test machines - keys.panel - ]; } diff --git a/launch/tests.nix b/launch/tests.nix index b86c7747..845e18b7 100644 --- a/launch/tests.nix +++ b/launch/tests.nix @@ -7,7 +7,10 @@ let }; }; tf = pkgs.callPackage ./tf.nix { }; - tfEnv = pkgs.callPackage ./tf-env.nix { }; + tfEnv = pkgs.callPackage ./tf-env.nix { + inherit lib pkgs; + dir = "launch/"; + }; in lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) { tf-validate = { diff --git a/launch/tf-env.nix b/launch/tf-env.nix index a29114f9..c06eb5d8 100644 --- a/launch/tf-env.nix +++ b/launch/tf-env.nix @@ -1,6 +1,7 @@ { lib, pkgs, + path, sources ? import ../npins, ... }: @@ -18,7 +19,7 @@ pkgs.stdenv.mkDerivation { ]; buildPhase = '' runHook preBuild - pushd launch/ + pushd ${path} # calculated pins echo '${lib.strings.toJSON sources}' > .npins.json # generate TF lock for nix's TF providers diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index 24426a96..f85ec407 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -29,7 +29,10 @@ let ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) (builtins.toFile "extra-settings.py" cfg.extra-settings) ]; - REPO_DIR = import ../../launch/tf-env.nix { inherit lib pkgs; }; + REPO_DIR = import ../../launch/tf-env.nix { + inherit lib pkgs; + dir = "launch/"; + }; }; python-environment = pkgs.python3.withPackages ( diff --git a/panel/nix/package.nix b/panel/nix/package.nix index 6808e37b..2cfd6582 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -70,7 +70,12 @@ python3.pkgs.buildPythonPackage { cp -v ${src}/manage.py $out/bin/manage.py chmod +x $out/bin/manage.py wrapProgram $out/bin/manage.py \ - --set REPO_DIR "${import ../../launch/tf-env.nix { inherit lib pkgs; }}" \ + --set REPO_DIR "${ + import ../../launch/tf-env.nix { + inherit lib pkgs; + dir = "launch/"; + } + }" \ --prefix PYTHONPATH : "$PYTHONPATH" ${lib.concatStringsSep "\n" ( map (file: "cp ${file.from} $out/${python3.sitePackages}/${file.to}") generated