From 58c1999fd2875e4f577e66c7e7fa63dde84690f2 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Thu, 1 May 2025 14:16:12 +0200 Subject: [PATCH] button works deployed late rebasing account for 285 --- .forgejo/workflows/ci.yaml | 6 + default.nix | 1 + infra/flake-part.nix | 19 +-- launch/.envrc | 10 ++ launch/.gitignore | 7 + launch/README.md | 28 +++ launch/default.nix | 34 ++++ launch/garage.nix | 34 ++++ launch/main.tf | 159 ++++++++++++++++++ launch/mastodon.nix | 17 ++ launch/options.nix | 54 ++++++ launch/peertube.nix | 20 +++ launch/pixelfed.nix | 16 ++ launch/resource.nix | 43 +++++ launch/shared.nix | 26 +++ launch/shell.nix | 1 + launch/tests.nix | 26 +++ launch/tf-env.nix | 34 ++++ launch/tf.nix | 24 +++ launch/variables.tf | 51 ++++++ machines/dev/fedi201/fedipanel.nix | 18 ++ panel/default.nix | 2 + panel/env.nix | 16 ++ panel/nix/configuration.nix | 21 +-- panel/nix/package.nix | 5 +- .../django-pydantic-field/default.nix | 1 + panel/src/panel/settings.py | 5 - panel/src/panel/views.py | 23 ++- 28 files changed, 654 insertions(+), 47 deletions(-) create mode 100644 launch/.envrc create mode 100644 launch/.gitignore create mode 100644 launch/README.md create mode 100644 launch/default.nix create mode 100644 launch/garage.nix create mode 100644 launch/main.tf create mode 100644 launch/mastodon.nix create mode 100644 launch/options.nix create mode 100644 launch/peertube.nix create mode 100644 launch/pixelfed.nix create mode 100644 launch/resource.nix create mode 100644 launch/shared.nix create mode 100644 launch/shell.nix create mode 100644 launch/tests.nix create mode 100644 launch/tf-env.nix create mode 100644 launch/tf.nix create mode 100644 launch/variables.tf create mode 100644 panel/env.nix diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 5015d407..7f8fe903 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -82,3 +82,9 @@ jobs: echo ~~~~~~~~~~~~~~~~~~~~~: $machine :~~~~~~~~~~~~~~~~~~~~~ nix build .#checks.x86_64-linux.nixosConfigurations-$machine done + + check-launch: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: cd launch && nix-build -A tests diff --git a/default.nix b/default.nix index 24b73cd2..e72a21ae 100644 --- a/default.nix +++ b/default.nix @@ -27,6 +27,7 @@ let ## Add a directory here if pre-commit hooks shouldn't apply to it. optout = [ "npins" + "launch/.terraform" ]; excludes = map (dir: "^${dir}/") optout; addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); diff --git a/infra/flake-part.nix b/infra/flake-part.nix index 34bc6e50..905846c4 100644 --- a/infra/flake-part.nix +++ b/infra/flake-part.nix @@ -8,7 +8,7 @@ }: let - inherit (builtins) readDir readFile fromJSON; + inherit (builtins) readDir; inherit (lib) attrNames mkOption @@ -160,14 +160,12 @@ let listSubdirectories = path: attrNames (filterAttrs (_: type: type == "directory") (readDir path)); machines = listSubdirectories ../machines/dev; - testMachines = listSubdirectories ../machines/operator; nixosConfigurations = - genAttrs machines (makeConfiguration false) - // genAttrs testMachines (makeConfiguration true); + genAttrs machines (makeConfiguration false); vmOptions = filterAttrs (_: value: value != null) # Filter out non-Fediversity VMs - (genAttrs machines (makeVmOptions false) // genAttrs testMachines (makeVmOptions true)); + (genAttrs machines (makeVmOptions false)); in { @@ -180,17 +178,6 @@ in ## - We add a “test” deployment with all test machines. nixops4Deployments = genAttrs machines makeDeployment' // { default = makeDeployment machines; - test = makeTestDeployment ( - fromJSON ( - let - env = builtins.getEnv "DEPLOYMENT"; - in - if env != "" then - env - else - builtins.trace "env var DEPLOYMENT not set, falling back to ../deployment/configuration.sample.json!" (readFile ../deployment/configuration.sample.json) - ) - ); }; flake = { inherit nixosConfigurations vmOptions; }; diff --git a/launch/.envrc b/launch/.envrc new file mode 100644 index 00000000..26ef376b --- /dev/null +++ b/launch/.envrc @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +# shellcheck shell=bash +if type -P lorri &>/dev/null; then + eval "$(lorri direnv --flake .)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use flake +fi diff --git a/launch/.gitignore b/launch/.gitignore new file mode 100644 index 00000000..9fd1eb48 --- /dev/null +++ b/launch/.gitignore @@ -0,0 +1,7 @@ +# generated +.auto.tfvars.json +.npins.json +.terraform/ +.terraform.lock.hcl +.terraform.tfstate.lock.info +terraform.tfstate* diff --git a/launch/README.md b/launch/README.md new file mode 100644 index 00000000..c0599bc8 --- /dev/null +++ b/launch/README.md @@ -0,0 +1,28 @@ +# service deployment + +deploys [NixOS](https://nixos.org/) templates using [OpenTofu](https://opentofu.org/). + +## requirements + +- [nix](https://nix.dev/) + +## usage + +### development + +before using other commands, if not using direnv: + +```sh +nix-shell +``` + +then to initialize, or after updating pins or TF providers: + +```sh +setup +``` + +## implementing + +proper documentation TODO. +until then, a reference implementation may be found in [`panel/`](https://git.fediversity.eu/Fediversity/Fediversity/src/branch/main/panel). diff --git a/launch/default.nix b/launch/default.nix new file mode 100644 index 00000000..ddda0b6a --- /dev/null +++ b/launch/default.nix @@ -0,0 +1,34 @@ +{ + system ? builtins.currentSystem, + sources ? import ../npins, + pkgs ? import sources.nixpkgs { inherit system; }, +}: +let + inherit (pkgs) lib; + setup = pkgs.writeScriptBin "setup" '' + echo '${lib.strings.toJSON sources}' > .npins.json + rm -f .terraform.lock.hcl + rm -rf .terraform/ + tofu init + ''; +in +{ + # shell for testing TF directly + shell = pkgs.mkShellNoCC { + packages = [ + (import ./tf.nix { inherit lib pkgs; }) + pkgs.jaq + setup + ]; + }; + + tests = pkgs.callPackage ./tests.nix { }; + + # re-export inputs so they can be overridden granularly + # (they can't be accessed from the outside any other way) + inherit + sources + system + pkgs + ; +} diff --git a/launch/garage.nix b/launch/garage.nix new file mode 100644 index 00000000..559ca37f --- /dev/null +++ b/launch/garage.nix @@ -0,0 +1,34 @@ +{ pkgs, ... }: +let + ## NOTE: All of these secrets are publicly available in this source file + ## and will end up in the Nix store. We don't care as they are only ever + ## used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +{ + fediversity = { + garage.enable = true; + pixelfed = pixelfedS3KeyConfig { inherit pkgs; }; + mastodon = mastodonS3KeyConfig { inherit pkgs; }; + peertube = peertubeS3KeyConfig { inherit pkgs; }; + }; +} diff --git a/launch/main.tf b/launch/main.tf new file mode 100644 index 00000000..93efc2cb --- /dev/null +++ b/launch/main.tf @@ -0,0 +1,159 @@ +locals { + system = "x86_64-linux" + # dependency paths pre-calculated from npins + pins = jsondecode(file("${path.root}/.npins.json")) + # nix path: expose pins, use nixpkgs in flake commands (`nix run`) + nix_path = "${join(":", [for name, path in local.pins : "${name}=${path}"])}:flake=${local.pins["nixpkgs"]}:flake" + # user-facing applications + application_configs = { + # FIXME: wrap applications at the interface to grab them in one go? + mastodon = { + cfg = var.mastodon + hostname = "test06" + } + pixelfed = { + cfg = var.pixelfed + hostname = "test04" + } + peertube = { + cfg = var.peertube + hostname = "test05" + } + } + # services shared between applications + peripherals = { for name, inst in { + garage = "test01" + } : name => { + hostname = inst + cfg = { + # enable if any user applications are enabled + enable = anytrue([for _, app in local.application_configs: try(app.cfg.enable, false)]) + } + } + } +} + +# hash of our code directory, used in dev to trigger re-deploy +# FIXME settle for pwd when in /nix/store? +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ..)\\\"}\""] +} + +# TF resource to build and deploy NixOS instances. +resource "terraform_data" "nixos" { + + for_each = {for name, inst in merge( + local.peripherals, + local.application_configs, + ) : name => inst if try(inst.cfg.enable, false)} + + # trigger rebuild/deploy if (FIXME?) any potentially used config/code changed, + # preventing these (20+s, build being bottleneck) when nothing changed. + # terraform-nixos separates these to only deploy if instantiate changed, + # yet building even then - which may be not as bad using deploy on remote. + # having build/deploy one resource reflects wanting to prevent no-op rebuilds + # over preventing (with less false positives) no-op deployments, + # as i could not find a way to do prevent no-op rebuilds without merging them: + # - generic resources cannot have outputs, while we want info from the instantiation (unless built on host?). + # - `data` always runs, which is slow for deploy and especially build. + triggers_replace = [ + data.external.hash.result, + var.domain, + var.initialUser, + local.system, + each.key, + each.value, + ] + + provisioner "local-exec" { + # directory to run the script from. we use the TF project root dir, + # here as a path relative from where TF is run from. + # note that absolute paths can cause false positives in triggers, + # so are generally discouraged in TF. + working_dir = path.root + environment = { + # nix path used on build, lets us refer to e.g. nixpkgs like `` + NIX_PATH = local.nix_path + } + # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs? + # TODO: build on target? + command = <<-EOF + set -euo pipefail + + # INSTANTIATE + command=( + nix-instantiate + --expr + 'let + os = import { + system = "${local.system}"; + configuration = { + # note interpolations here TF ones + imports = [ + # shared NixOS config + ${path.root}/shared.nix + # FIXME: separate template options by service + ${path.root}/options.nix + # for service `mastodon` import `mastodon.nix` + ${path.root}/${each.key}.nix + # FIXME: get VM details from TF + ${path.root}/../infra/test-machines/${each.value.hostname} + ]; + # nix path for debugging + nix.nixPath = [ "${local.nix_path}" ]; + ## FIXME: switch root authentication to users with password-less sudo, see #24 + users.users.root.openssh.authorizedKeys.keys = let + keys = import ../keys; + in attrValues keys.contributors ++ [ + # allow our panel vm access to the test machines + keys.panel + ]; + } // + # template parameters passed in from TF thru json + builtins.fromJSON "${replace(jsonencode({ + terraform = { + domain = var.domain + hostname = each.value.hostname + initialUser = var.initialUser + } + }), "\"", "\\\"")}"; + }; + in + # info we want to get back out + { + substituters = builtins.concatStringsSep " " os.config.nix.settings.substituters; + trusted_public_keys = builtins.concatStringsSep " " os.config.nix.settings.trusted-public-keys; + drv_path = os.config.system.build.toplevel.drvPath; + out_path = os.config.system.build.toplevel; + }' + ) + # instantiate the config in /nix/store + "$${command[@]}" -A out_path + # get the other info + json="$("$${command[@]}" --eval --strict --json)" + + # DEPLOY + declare substituters trusted_public_keys drv_path + # set our variables using the json object + eval "export $(echo $json | jaq -r 'to_entries | map("\(.key)=\(.value)") | @sh')" + # FIXME: de-hardcode domain + host="root@${each.value.hostname}.abundos.eu" # FIXME: #24 + buildArgs=( + --option extra-binary-caches https://cache.nixos.org/ + --option substituters $substituters + --option trusted-public-keys $trusted_public_keys + ) + sshOpts=( + -o BatchMode=yes + -o StrictHostKeyChecking=no + ) + # get the realized derivation to deploy + outPath=$(nix-store --realize "$drv_path" "$${buildArgs[@]}") + # deploy the config by nix-copy-closure + NIX_SSHOPTS="$${sshOpts[*]}" nix-copy-closure --to "$host" "$outPath" --gzip --use-substitutes + # switch the remote host to the config + ssh "$${sshOpts[@]}" "$host" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; $outPath/bin/switch-to-configuration switch" + EOF + } +} diff --git a/launch/mastodon.nix b/launch/mastodon.nix new file mode 100644 index 00000000..26682f50 --- /dev/null +++ b/launch/mastodon.nix @@ -0,0 +1,17 @@ +{ pkgs, ... }: +let + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; +in +{ + fediversity = { + mastodon = mastodonS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + temp.cores = 1; # FIXME: should come from NixOps4 eventually + }; +} diff --git a/launch/options.nix b/launch/options.nix new file mode 100644 index 00000000..22344017 --- /dev/null +++ b/launch/options.nix @@ -0,0 +1,54 @@ +# TODO: could (part of) this be generated somehow? c.f #275 +{ + lib, + ... +}: +let + inherit (lib) types mkOption; + inherit (types) str enum submodule; +in +{ + options.terraform = { + domain = mkOption { + type = enum [ + "fediversity.net" + ]; + description = '' + Apex domain under which the services will be deployed. + ''; + default = "fediversity.net"; + }; + hostname = mkOption { + type = str; + description = '' + Internal name of the host, e.g. test01 + ''; + }; + initialUser = mkOption { + description = '' + Some services require an initial user to access them. + This option sets the credentials for such an initial user. + ''; + type = submodule { + options = { + displayName = mkOption { + type = str; + description = "Display name of the user"; + }; + username = mkOption { + type = str; + description = "Username for login"; + }; + email = mkOption { + type = str; + description = "User's email address"; + }; + password = mkOption { + type = str; + description = "Password for login"; + }; + }; + }; + }; + }; +} diff --git a/launch/peertube.nix b/launch/peertube.nix new file mode 100644 index 00000000..4124568a --- /dev/null +++ b/launch/peertube.nix @@ -0,0 +1,20 @@ +{ pkgs, ... }: +let + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; +in +{ + fediversity = { + peertube = peertubeS3KeyConfig { inherit pkgs; } // { + enable = true; + ## NOTE: Only ever used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24"; + }; + }; +} diff --git a/launch/pixelfed.nix b/launch/pixelfed.nix new file mode 100644 index 00000000..75790409 --- /dev/null +++ b/launch/pixelfed.nix @@ -0,0 +1,16 @@ +{ pkgs, ... }: +let + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +{ + fediversity = { + pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + }; +} diff --git a/launch/resource.nix b/launch/resource.nix new file mode 100644 index 00000000..ec719e20 --- /dev/null +++ b/launch/resource.nix @@ -0,0 +1,43 @@ +{ + lib, + config, + ... +}: + +let + inherit (lib) attrValues elem mkDefault; + inherit (lib.attrsets) concatMapAttrs optionalAttrs; + inherit (lib.strings) removeSuffix; + + secretsPrefix = ../secrets; + secrets = import (secretsPrefix + "/secrets.nix"); + keys = import ../keys; + +in +{ + fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name}; + + ## The configuration of the machine. We strive to keep in this file only the + ## options that really need to be injected from the resource. Everything else + ## should go into the `./nixos` subdirectory. + imports = [ + ../infra/common/options.nix + ../infra/common/nixos + ]; + + ## Read all the secrets, filter the ones that are supposed to be readable + ## with this host's public key, and add them correctly to the configuration + ## as `age.secrets..file`. + age.secrets = concatMapAttrs ( + name: secret: + optionalAttrs (elem config.fediversityVm.hostPublicKey secret.publicKeys) { + ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}"; + } + ) secrets; + + ## FIXME: switch root authentication to users with password-less sudo, see #24 + users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors ++ [ + # allow our panel vm access to the test machines + keys.panel + ]; +} diff --git a/launch/shared.nix b/launch/shared.nix new file mode 100644 index 00000000..46d6ccc5 --- /dev/null +++ b/launch/shared.nix @@ -0,0 +1,26 @@ +{ + pkgs, + config, + ... +}: +let + inherit (config.terraform) hostname domain initialUser; +in +{ + imports = [ + + + ../services/fediversity + ./resource.nix + ]; + fediversityVm.name = hostname; + fediversity = { + inherit domain; + temp.initialUser = { + inherit (initialUser) username email displayName; + # FIXME: disgusting, but nvm, this is going to be replaced by + # proper central authentication at some point + passwordFile = pkgs.writeText "password" initialUser.password; + }; + }; +} diff --git a/launch/shell.nix b/launch/shell.nix new file mode 100644 index 00000000..a6bdf202 --- /dev/null +++ b/launch/shell.nix @@ -0,0 +1 @@ +(import ./. { }).shell diff --git a/launch/tests.nix b/launch/tests.nix new file mode 100644 index 00000000..b86c7747 --- /dev/null +++ b/launch/tests.nix @@ -0,0 +1,26 @@ +{ lib, pkgs }: +let + defaults = { + virtualisation = { + memorySize = 2048; + cores = 2; + }; + }; + tf = pkgs.callPackage ./tf.nix { }; + tfEnv = pkgs.callPackage ./tf-env.nix { }; +in +lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) { + tf-validate = { + inherit defaults; + nodes.server = { + environment.systemPackages = [ + tf + tfEnv + ]; + }; + testScript = '' + server.wait_for_unit("multi-user.target") + server.succeed("${lib.getExe tf} -chdir='${tfEnv}/launch' validate") + ''; + }; +} diff --git a/launch/tf-env.nix b/launch/tf-env.nix new file mode 100644 index 00000000..a29114f9 --- /dev/null +++ b/launch/tf-env.nix @@ -0,0 +1,34 @@ +{ + lib, + pkgs, + sources ? import ../npins, + ... +}: +pkgs.stdenv.mkDerivation { + name = "tf-repo"; + src = + with lib.fileset; + toSource { + root = ../.; + # don't copy ignored files + fileset = intersection (gitTracked ../.) ../.; + }; + buildInputs = [ + (import ./tf.nix { inherit lib pkgs; }) + ]; + buildPhase = '' + runHook preBuild + pushd launch/ + # calculated pins + echo '${lib.strings.toJSON sources}' > .npins.json + # generate TF lock for nix's TF providers + tofu init -input=false + popd + runHook postBuild + ''; + installPhase = '' + runHook preInstall + cp -r . $out + runHook postInstall + ''; +} diff --git a/launch/tf.nix b/launch/tf.nix new file mode 100644 index 00000000..43a59298 --- /dev/null +++ b/launch/tf.nix @@ -0,0 +1,24 @@ +# FIXME: use overlays so this gets imported just once? +{ + lib, + pkgs, + ... +}: +let + tofuProvider = + provider: + provider.override (oldArgs: { + provider-source-address = + lib.replaceStrings [ "https://registry.terraform.io/providers" ] [ "registry.opentofu.org" ] + oldArgs.homepage; + }); + tf = pkgs.opentofu; + tfPlugins = ( + p: [ + p.external + ] + ); +in +# tf.withPlugins tfPlugins +# https://github.com/NixOS/nixpkgs/pull/358522 +tf.withPlugins (p: pkgs.lib.lists.map tofuProvider (tfPlugins p)) diff --git a/launch/variables.tf b/launch/variables.tf new file mode 100644 index 00000000..05f1b2f6 --- /dev/null +++ b/launch/variables.tf @@ -0,0 +1,51 @@ +# TODO: (partially) generate, say from nix modules, c.f. #275 + +variable "domain" { + type = string + default = "fediversity.net" +} + +variable "mastodon" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "pixelfed" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "peertube" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "initialUser" { + type = object({ + displayName = string + username = string + email = string + # TODO: mark (nested) credentials as sensitive + # https://discuss.hashicorp.com/t/is-it-possible-to-mark-an-attribute-of-an-object-as-sensitive/24649/2 + password = string + }) + # FIXME: remove default when the form provides this value, see #285 + default = { + displayName = "Testy McTestface" + username = "test" + email = "test@test.com" + password = "testtest" + } +} diff --git a/machines/dev/fedi201/fedipanel.nix b/machines/dev/fedi201/fedipanel.nix index 494212de..bd1c25ce 100644 --- a/machines/dev/fedi201/fedipanel.nix +++ b/machines/dev/fedi201/fedipanel.nix @@ -39,6 +39,24 @@ in enable = true; production = true; domain = "demo.fediversity.eu"; + # FIXME: make it work without this duplication + settings = + let + cfg = config.services.${name}; + in + { + STATIC_ROOT = "/var/lib/${name}/static"; + DEBUG = false; + ALLOWED_HOSTS = [ + cfg.domain + cfg.host + "localhost" + "[::1]" + ]; + CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ]; + COMPRESS_OFFLINE = true; + LIBSASS_OUTPUT_STYLE = "compressed"; + }; secrets = { SECRET_KEY = config.age.secrets.panel-secret-key.path; }; diff --git a/panel/default.nix b/panel/default.nix index a8f716c9..f531b5cd 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -32,6 +32,8 @@ in NPINS_DIRECTORY = toString ../npins; CREDENTIALS_DIRECTORY = toString ./.credentials; DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; + # locally: use a fixed relative reference, so we can use our newest files without copying to the store + REPO_DIR = toString ../.; }; shellHook = '' ${lib.concatStringsSep "\n" ( diff --git a/panel/env.nix b/panel/env.nix new file mode 100644 index 00000000..b4ecffc5 --- /dev/null +++ b/panel/env.nix @@ -0,0 +1,16 @@ +{ + lib, + pkgs, + ... +}: +{ + BIN_PATH = lib.makeBinPath [ + pkgs.lix + pkgs.bash + pkgs.coreutils + pkgs.openssh + pkgs.git + pkgs.jaq # tf + (import ../launch/tf.nix { inherit lib pkgs; }) + ]; +} diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index 07dd250d..f59b3243 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -24,13 +24,12 @@ let package = pkgs.callPackage ./package.nix { }; environment = { - DEPLOYMENT_FLAKE = cfg.deployment.flake; - DEPLOYMENT_NAME = cfg.deployment.name; DATABASE_URL = "sqlite:////var/lib/${name}/db.sqlite3"; USER_SETTINGS_FILE = pkgs.concatText "configuration.py" [ ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) (builtins.toFile "extra-settings.py" cfg.extra-settings) ]; + REPO_DIR = import ../../launch/tf-env.nix { inherit lib pkgs; }; }; python-environment = pkgs.python3.withPackages ( @@ -149,23 +148,6 @@ in ''; default = pkgs.nixops4; }; - - deployment = { - flake = mkOption { - type = types.path; - default = ../..; - description = '' - The path to the flake containing the deployment. This is used to run the deployment button. - ''; - }; - name = mkOption { - type = types.str; - default = "test"; - description = '' - The name of the deployment within the flake. - ''; - }; - }; }; config = mkIf cfg.enable { @@ -211,6 +193,7 @@ in after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = [ + pkgs.openssh python-environment manage-service diff --git a/panel/nix/package.nix b/panel/nix/package.nix index 76d726d0..6f99162b 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -1,5 +1,6 @@ { lib, + pkgs, sqlite, python3, python3Packages, @@ -98,7 +99,9 @@ python3.pkgs.buildPythonPackage { mkdir -p $out/bin cp -v ${src}/manage.py $out/bin/manage.py chmod +x $out/bin/manage.py - wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH" + wrapProgram $out/bin/manage.py \ + --set REPO_DIR "${import ../../launch/tf-env.nix { inherit lib pkgs; }}" \ + --prefix PYTHONPATH : "$PYTHONPATH" ${lib.concatStringsSep "\n" ( map (file: "cp ${file.from} $out/${python3.sitePackages}/${file.to}") generated )} diff --git a/panel/nix/python-packages/django-pydantic-field/default.nix b/panel/nix/python-packages/django-pydantic-field/default.nix index a1779961..7df1cd3c 100644 --- a/panel/nix/python-packages/django-pydantic-field/default.nix +++ b/panel/nix/python-packages/django-pydantic-field/default.nix @@ -1,3 +1,4 @@ +# TODO upstream, see #248 { lib, buildPythonPackage, diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index 5a555495..2783a06b 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -240,8 +240,3 @@ if user_settings_file is not None: # TODO(@fricklerhandwerk): # The correct thing to do here would be using a helper function such as with `get_secret()` that will catch the exception and explain what's wrong and where to put the right values. # Replacing the `USER_SETTINGS_FILE` mechanism following the comment there would probably be a good thing. - -# Path of the root flake to trigger nixops from, see #94, and name of the -# deployment. -deployment_flake = env["DEPLOYMENT_FLAKE"] -deployment_name = env["DEPLOYMENT_NAME"] diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 2f603002..4d6c23cd 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -1,6 +1,7 @@ from enum import Enum import json import subprocess +import logging import os from django.urls import reverse_lazy @@ -19,6 +20,8 @@ from pydantic import BaseModel from panel import models, settings from panel.configuration import schema +logger = logging.getLogger(__name__) + class Index(TemplateView): template_name = 'index.html' @@ -90,20 +93,28 @@ class DeploymentStatus(ConfigurationForm): def deployment(self, config: BaseModel): env = { "PATH": os.environ.get("PATH"), + # "TF_LOG": "info", + } | { # pass in form info to our deployment - "DEPLOYMENT": config.json() + # FIXME: ensure sensitive info is protected + f"TF_VAR_{k}": v if isinstance(v, str) else json.dumps(v) for k, v in json.loads(config.model_dump_json()).items() } + logger.info("env: %s", env) + cwd = f"{settings.repo_dir}/launch" cmd = [ - "nixops4", + "tofu", + # f"-chdir={cwd}", "apply", - settings.deployment_name, - "--show-trace", - "--no-interactive", + f"-state={cwd}/terraform.tfstate", # FIXME: separate users' state + "--auto-approve", + "-lock=false", + "-parallelism=1" # limit OOM risk ] deployment_result = subprocess.run( cmd, - cwd = settings.deployment_flake, + cwd = cwd, env = env, stderr = subprocess.STDOUT, ) + logger.debug("deployment_result: %s", deployment_result) return deployment_result, config