From 3eb6d9321606304540cb1c9a3190bfe5b7669340 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 24 Mar 2025 10:41:07 +0100 Subject: [PATCH] tf --- infra/machines/fedi201/fedipanel.nix | 18 +++ launch/.envrc | 10 ++ launch/.gitignore | 3 + launch/.terraform.lock.hcl | 16 +++ launch/.terraform/modules/deploy | 1 + launch/.terraform/modules/garage.deploy | 1 + launch/README.md | 9 ++ launch/default.nix | 22 ++++ launch/flake.lock | 158 ++++++++++++++++++++++++ launch/flake.nix | 125 +++++++++++++++++++ launch/main.tf | 84 +++++++++++++ launch/pass-ssh-key.sh | 15 +++ launch/resource.nix | 41 ++++++ launch/shell.nix | 1 + launch/tf.nix | 25 ++++ launch/vm/main.tf | 46 +++++++ npins/sources.json | 12 ++ panel/env.nix | 11 +- panel/nix/package.nix | 6 +- panel/src/panel/settings.py | 69 ++++++++++- panel/src/panel/views.py | 42 ++++--- 21 files changed, 692 insertions(+), 23 deletions(-) create mode 100644 launch/.envrc create mode 100644 launch/.gitignore create mode 100644 launch/.terraform.lock.hcl create mode 160000 launch/.terraform/modules/deploy create mode 160000 launch/.terraform/modules/garage.deploy create mode 100644 launch/README.md create mode 100644 launch/default.nix create mode 100644 launch/flake.lock create mode 100644 launch/flake.nix create mode 100644 launch/main.tf create mode 100755 launch/pass-ssh-key.sh create mode 100644 launch/resource.nix create mode 100644 launch/shell.nix create mode 100644 launch/tf.nix create mode 100644 launch/vm/main.tf diff --git a/infra/machines/fedi201/fedipanel.nix b/infra/machines/fedi201/fedipanel.nix index 5c4236fc..b49471bb 100644 --- a/infra/machines/fedi201/fedipanel.nix +++ b/infra/machines/fedi201/fedipanel.nix @@ -37,6 +37,24 @@ in enable = true; production = true; domain = "demo.fediversity.eu"; + # FIXME: make it work without this duplication + settings = + let + cfg = config.services.${name}; + in + { + STATIC_ROOT = "/var/lib/${name}/static"; + DEBUG = false; + ALLOWED_HOSTS = [ + cfg.domain + cfg.host + "localhost" + "[::1]" + ]; + CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ]; + COMPRESS_OFFLINE = true; + LIBSASS_OUTPUT_STYLE = "compressed"; + }; secrets = { SECRET_KEY = config.age.secrets.panel-secret-key.path; }; diff --git a/launch/.envrc b/launch/.envrc new file mode 100644 index 00000000..26ef376b --- /dev/null +++ b/launch/.envrc @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +# shellcheck shell=bash +if type -P lorri &>/dev/null; then + eval "$(lorri direnv --flake .)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use flake +fi diff --git a/launch/.gitignore b/launch/.gitignore new file mode 100644 index 00000000..0f8ba9b3 --- /dev/null +++ b/launch/.gitignore @@ -0,0 +1,3 @@ +.terraform/ +.terraform.tfstate.lock.info +terraform.tfstate* diff --git a/launch/.terraform.lock.hcl b/launch/.terraform.lock.hcl new file mode 100644 index 00000000..f90c28b6 --- /dev/null +++ b/launch/.terraform.lock.hcl @@ -0,0 +1,16 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/external" { + version = "2.3.4" + hashes = [ + "h1:F45RHS5NlHBWkfcW61rwkA7Z2P+p3Rxvq75XMhNeFHE=", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:P9gA/YsRU4Q+STK6EkCOjTHYUop/+1LfIbX7QMNHfPk=", + ] +} diff --git a/launch/.terraform/modules/deploy b/launch/.terraform/modules/deploy new file mode 160000 index 00000000..657cc08c --- /dev/null +++ b/launch/.terraform/modules/deploy @@ -0,0 +1 @@ +Subproject commit 657cc08c2a22cf6500de70df5519ccb194449f0f diff --git a/launch/.terraform/modules/garage.deploy b/launch/.terraform/modules/garage.deploy new file mode 160000 index 00000000..657cc08c --- /dev/null +++ b/launch/.terraform/modules/garage.deploy @@ -0,0 +1 @@ +Subproject commit 657cc08c2a22cf6500de70df5519ccb194449f0f diff --git a/launch/README.md b/launch/README.md new file mode 100644 index 00000000..a30dbbfa --- /dev/null +++ b/launch/README.md @@ -0,0 +1,9 @@ +# service deployment + +## usage + +### updating TF modules + +```sh +echo "{\"nixos-anywhere\": $(nix-instantiate --eval --json -E '(import ../npins).nixos-anywhere.outPath')}" > .auto.tfvars.json +``` diff --git a/launch/default.nix b/launch/default.nix new file mode 100644 index 00000000..9a533dd8 --- /dev/null +++ b/launch/default.nix @@ -0,0 +1,22 @@ +{ + system ? builtins.currentSystem, + sources ? import ../npins, + pkgs ? import sources.nixpkgs { + inherit system; + }, +}@args: +let + inherit (pkgs) lib; +in +{ + shell = pkgs.mkShellNoCC { + packages = [ + pkgs.npins + pkgs.jq # implicit dep of nixos-anywhere TF: https://github.com/nix-community/nixos-anywhere/issues/416 + (import ./tf.nix { inherit lib pkgs; }) + ]; + }; +} +# re-export inputs so they can be overridden granularly +# (they can't be accessed from the outside any other way) +// args diff --git a/launch/flake.lock b/launch/flake.lock new file mode 100644 index 00000000..124c30b2 --- /dev/null +++ b/launch/flake.lock @@ -0,0 +1,158 @@ +{ + "nodes": { + "agenix": { + "inputs": { + "darwin": "darwin", + "home-manager": "home-manager", + "nixpkgs": "nixpkgs", + "systems": "systems" + }, + "locked": { + "lastModified": 1736955230, + "narHash": "sha256-uenf8fv2eG5bKM8C/UvFaiJMZ4IpUFaQxk9OH5t/1gA=", + "owner": "ryantm", + "repo": "agenix", + "rev": "e600439ec4c273cf11e06fe4d9d906fb98fa097c", + "type": "github" + }, + "original": { + "owner": "ryantm", + "repo": "agenix", + "type": "github" + } + }, + "darwin": { + "inputs": { + "nixpkgs": [ + "agenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1700795494, + "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=", + "owner": "lnl7", + "repo": "nix-darwin", + "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d", + "type": "github" + }, + "original": { + "owner": "lnl7", + "ref": "master", + "repo": "nix-darwin", + "type": "github" + } + }, + "disko": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1741786315, + "narHash": "sha256-VT65AE2syHVj6v/DGB496bqBnu1PXrrzwlw07/Zpllc=", + "owner": "nix-community", + "repo": "disko", + "rev": "0d8c6ad4a43906d14abd5c60e0ffe7b587b213de", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "home-manager": { + "inputs": { + "nixpkgs": [ + "agenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1703113217, + "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1703013332, + "narHash": "sha256-+tFNwMvlXLbJZXiMHqYq77z/RfmpfpiI3yjL6o/Zo9M=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "54aac082a4d9bb5bbc5c4e899603abfb76a3f6d6", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1741402956, + "narHash": "sha256-y2hByvBM03s9T2fpeLjW6iprbxnhV9mJMmSwCHc41ZQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ed0b1881565c1ffef490c10d663d4f542031dad3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1742753982, + "narHash": "sha256-WYryX6lCrmh4AaEoBHA1zwTTs6vPtevT3/ywGyzz4SI=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "ec9c54e7a9feec999aa34a15781080bed5082306", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "release-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "agenix": "agenix", + "disko": "disko", + "nixpkgs": "nixpkgs_3" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/launch/flake.nix b/launch/flake.nix new file mode 100644 index 00000000..9d126b9e --- /dev/null +++ b/launch/flake.nix @@ -0,0 +1,125 @@ +{ + inputs = { + agenix.url = "github:ryantm/agenix"; + disko.url = "github:nix-community/disko"; + nixpkgs.url = "github:nixos/nixpkgs/release-24.11"; + }; + outputs = + inputs@{ nixpkgs, ... }: + let + system = "x86_64-linux"; + inherit (nixpkgs) lib; + in + { + nixosConfigurations = + let + ## NOTE: All of these secrets are publicly available in this source file + ## and will end up in the Nix store. We don't care as they are only ever + ## used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; + in + lib.mapAttrs + ( + _: module: + lib.nixosSystem { + inherit system; + specialArgs = { inherit system inputs; }; + modules = [ + inputs.disko.nixosModules.default + inputs.agenix.nixosModules.default + ../services/fediversity + ./resource.nix + module + { + nixpkgs = { inherit system; }; + } + ( + { pkgs, terraform, ... }: + let + inherit (terraform) hostname; + in + { + imports = [ + # FIXME: get VM details from TF + ../infra/test-machines/${hostname} + ]; + fediversityVm.name = hostname; + fediversity = { + inherit (terraform) domain; + temp.initialUser = { + inherit (terraform.initialUser) username email displayName; + # FIXME: disgusting, but nvm, this is going to be replaced by + # proper central authentication at some point + passwordFile = pkgs.writeText "password" terraform.initialUser.password; + }; + }; + } + ) + ]; + } + ) + { + garage = + { pkgs, ... }: + { + fediversity = { + garage.enable = true; + pixelfed = pixelfedS3KeyConfig { inherit pkgs; }; + mastodon = mastodonS3KeyConfig { inherit pkgs; }; + peertube = peertubeS3KeyConfig { inherit pkgs; }; + }; + }; + mastodon = + { pkgs, ... }: + { + fediversity = { + mastodon = mastodonS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + temp.cores = 1; # FIXME: should come from NixOps4 eventually + }; + }; + peertube = + { pkgs, ... }: + { + fediversity = { + peertube = peertubeS3KeyConfig { inherit pkgs; } // { + enable = true; + ## NOTE: Only ever used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24"; + }; + }; + }; + pixelfed = + { pkgs, ... }: + { + fediversity = { + pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + }; + }; + }; + }; +} diff --git a/launch/main.tf b/launch/main.tf new file mode 100644 index 00000000..2e4aabcc --- /dev/null +++ b/launch/main.tf @@ -0,0 +1,84 @@ +variable "domain" { + type = string + default = "fediversity.net" +} + +variable "mastodon" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "pixelfed" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "peertube" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "initialUser" { + type = object({ + displayName = string + username = string + email = string + # TODO: mark (nested) credentials as sensitive + # https://discuss.hashicorp.com/t/is-it-possible-to-mark-an-attribute-of-an-object-as-sensitive/24649/2 + password = string + }) + default = { + displayName = "Testy McTestface" + username = "test" + email = "test@test.com" + password = "testtest" + } +} + +# module "garage" { +# source = "./vm" +# count = var.mastodon.enable || var.pixelfed.enable || var.peertube.enable ? 1 : 0 +# domain = var.domain +# hostname = "test01" +# config = "garage" +# initialUser = var.initialUser +# } + +module "mastodon" { + source = "./vm" + count = var.mastodon.enable ? 1 : 0 + domain = var.domain + hostname = "test02" + config = "mastodon" + initialUser = var.initialUser +} + +module "pixelfed" { + source = "./vm" + count = var.pixelfed.enable ? 1 : 0 + domain = var.domain + hostname = "test04" + config = "pixelfed" + initialUser = var.initialUser +} + +module "peertube" { + source = "./vm" + count = var.peertube.enable ? 1 : 0 + domain = var.domain + hostname = "test03" + config = "peertube" + initialUser = var.initialUser +} diff --git a/launch/pass-ssh-key.sh b/launch/pass-ssh-key.sh new file mode 100755 index 00000000..80f17dec --- /dev/null +++ b/launch/pass-ssh-key.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +export host="$host" + +mkdir -p etc/ssh + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +for keyname in ssh_host_ed25519_key ssh_host_ed25519_key.pub; do + if [[ $keyname == *.pub ]]; then + umask 0133 + else + umask 0177 + fi + cp "$SCRIPT_DIR/../infra/test-machines/${host}/${keyname}" ./etc/ssh/${keyname} +done diff --git a/launch/resource.nix b/launch/resource.nix new file mode 100644 index 00000000..04811271 --- /dev/null +++ b/launch/resource.nix @@ -0,0 +1,41 @@ +{ + lib, + config, + ... +}: + +let + inherit (lib) attrValues elem mkDefault; + inherit (lib.attrsets) concatMapAttrs optionalAttrs; + inherit (lib.strings) removeSuffix; + + secretsPrefix = ../secrets; + secrets = import (secretsPrefix + "/secrets.nix"); + keys = import ../keys; + +in +{ + fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name}; + + ## The configuration of the machine. We strive to keep in this file only the + ## options that really need to be injected from the resource. Everything else + ## should go into the `./nixos` subdirectory. + imports = [ + ../infra/common/options.nix + ../infra/common/nixos + ]; + + ## Read all the secrets, filter the ones that are supposed to be readable + ## with this host's public key, and add them correctly to the configuration + ## as `age.secrets..file`. + age.secrets = concatMapAttrs ( + name: secret: + optionalAttrs (elem config.fediversityVm.hostPublicKey secret.publicKeys) { + ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}"; + } + ) secrets; + + ## FIXME: Remove direct root authentication once the NixOps4 NixOS provider + ## supports users with password-less sudo. + users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors; +} diff --git a/launch/shell.nix b/launch/shell.nix new file mode 100644 index 00000000..a6bdf202 --- /dev/null +++ b/launch/shell.nix @@ -0,0 +1 @@ +(import ./. { }).shell diff --git a/launch/tf.nix b/launch/tf.nix new file mode 100644 index 00000000..14ad72a7 --- /dev/null +++ b/launch/tf.nix @@ -0,0 +1,25 @@ +# FIXME: use overlays so this gets imported just once? +{ + lib, + pkgs, + ... +}: +let + tofuProvider = + provider: + provider.override (oldArgs: { + provider-source-address = + lib.replaceStrings [ "https://registry.terraform.io/providers" ] [ "registry.opentofu.org" ] + oldArgs.homepage; + }); + tf = pkgs.opentofu; + tfPlugins = ( + p: [ + p.null + p.external + ] + ); +in +# tf.withPlugins tfPlugins +# https://github.com/NixOS/nixpkgs/pull/358522 +tf.withPlugins (p: pkgs.lib.lists.map tofuProvider (tfPlugins p)) diff --git a/launch/vm/main.tf b/launch/vm/main.tf new file mode 100644 index 00000000..c616d0e0 --- /dev/null +++ b/launch/vm/main.tf @@ -0,0 +1,46 @@ +variable "domain" { + type = string +} + +variable "hostname" { + type = string +} + +variable "config" { + type = string +} + +variable "initialUser" { + type = object({ + displayName = string + username = string + password = string + email = string + }) +} + +module "deploy" { + # source = "github.com/nix-community/nixos-anywhere//terraform/all-in-one" + source = "github.com/KiaraGrouwstra/nixos-anywhere?ref=special-args-nested-flake-fixed//terraform/all-in-one" + nixos_system_attr = ".#nixosConfigurations.${var.config}.config.system.build.toplevel" + nixos_partitioner_attr = ".#nixosConfigurations.${var.config}.config.system.build.diskoScriptNoDeps" + # when instance id changes, it will trigger a reinstall + instance_id = var.hostname + target_user = "kiara" + target_host = "${var.hostname}.abundos.eu" + extra_files_script = "${path.module}/../pass-ssh-key.sh" + extra_environment = { + host = var.hostname + } + special_args = { + terraform = { + domain = var.domain + hostname = var.hostname + initialUser = var.initialUser + } + } + nix_options = { + show-trace = true + } + # build_on_remote = true +} diff --git a/npins/sources.json b/npins/sources.json index 45efd6b4..4701917c 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -27,6 +27,18 @@ "url": "https://github.com/nix-community/nix-unit/archive/2071bbb765681ac3d8194ec560c8b27ff2a3b541.tar.gz", "hash": "0blz1kcmn9vnr9q3iqp2mv13hv3pdccljmmc54f8j7ybf5v0wgmp" }, + "nixos-anywhere": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "KiaraGrouwstra", + "repo": "nixos-anywhere" + }, + "branch": "special-args-nested-flake-fixed", + "revision": "bbf54831f177b4ae52b55758ac725583617c39f9", + "url": "https://github.com/KiaraGrouwstra/nixos-anywhere/archive/bbf54831f177b4ae52b55758ac725583617c39f9.tar.gz", + "hash": "11m8r632yp4s8sy106zk7a1inp44gvwq6hinjf2pczpr6a5hxanx" + }, "nixpkgs": { "type": "Channel", "name": "nixpkgs-unstable", diff --git a/panel/env.nix b/panel/env.nix index 07ce4193..87805588 100644 --- a/panel/env.nix +++ b/panel/env.nix @@ -7,12 +7,15 @@ let inherit (builtins) toString; in { + # locally: use a fixed relative reference, so we can use our newest files without copying to the store REPO_DIR = toString ../.; - # explicitly use nix, as e.g. lix does not have configurable-impure-env BIN_PATH = lib.makeBinPath [ - # explicitly use nix, as e.g. lix does not have configurable-impure-env - pkgs.nix - # nixops error maybe due to our flake git hook: executing 'git': No such file or directory + pkgs.lix + pkgs.bash + pkgs.coreutils + pkgs.openssh pkgs.git + pkgs.jq # implicit dep of nixos-anywhere TF: https://github.com/nix-community/nixos-anywhere/issues/416 + (import ../launch/tf.nix { inherit lib pkgs; }) ]; } diff --git a/panel/nix/package.nix b/panel/nix/package.nix index 9337887c..2f94fa8f 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -11,7 +11,7 @@ let root = ../src; fileset = intersection (gitTracked ../../.) ../src; }; - pyproject = with lib; fromTOML pyproject-toml; + pyproject = fromTOML pyproject-toml; # TODO: define this globally name = "panel"; # TODO: we may want this in a file so it's easier to read statically @@ -58,7 +58,9 @@ python3.pkgs.buildPythonPackage { mkdir -p $out/bin cp -v ${src}/manage.py $out/bin/manage.py chmod +x $out/bin/manage.py - wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH" + wrapProgram $out/bin/manage.py \ + --set REPO_DIR "${../..}" \ + --prefix PYTHONPATH : "$PYTHONPATH" cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/ ''; } diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index b270612c..01c88963 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -10,7 +10,9 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/4.2/ref/settings/ """ +import re import sys +import subprocess import os import importlib.util import dj_database_url @@ -18,6 +20,8 @@ import dj_database_url from os import environ as env from pathlib import Path +STORE_PATTERN = re.compile("^/nix/store/[^/]+$") + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -171,6 +175,54 @@ COMPRESS_PRECOMPILERS = [ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", + }, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", + }, + }, + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", + } + }, + "handlers": { + "console": { + "level": "INFO", + # "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + }, + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", + }, + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, + }, + "loggers": { + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", + }, + "django.server": { + "handlers": ["django.server"], + "level": "INFO", + "propagate": False, + }, + }, +} + # Customization via user settings # This must be at the end, as it must be able to override the above # TODO(@fricklerhandwerk): @@ -186,6 +238,21 @@ if user_settings_file is not None: sys.modules["user_settings"] = module from user_settings import * # noqa: F403 # pyright: ignore [reportMissingImports] +def handle_double_hash(orig_path: str) -> str: + """Convert store paths to account for potentially double hashes. + + c.f. https://github.com/NixOS/nix/issues/10627 + """ + # local paths can stay as-is + if not STORE_PATTERN.match(orig_path): + return orig_path + # for /nix/store paths, account for double hashing + orig_name = orig_path.split("/")[-1] + cmd = ["find", "/nix/store/", "-maxdepth", "1", "-name", f"*{orig_name}"] + process = subprocess.run(cmd, capture_output=True) + fixed_name = process.stdout.decode("utf-8").split("\n")[0] + return fixed_name + # non-Django application settings # TODO(@fricklerhandwerk): @@ -196,4 +263,4 @@ if user_settings_file is not None: bin_path=env['BIN_PATH'] # path of the root flake to trigger nixops from, see #94. # to deploy this should be specified, for dev just use a relative path. -repo_dir = env["REPO_DIR"] +repo_dir = handle_double_hash(env["REPO_DIR"]) diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 5c423fc6..b9abf8cc 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -1,5 +1,6 @@ from enum import Enum import json +from os.path import expanduser import subprocess import os @@ -134,25 +135,34 @@ class DeploymentStatus(ConfigurationForm): }, } # serialize back and forth now we still need to manually inject the dummy user - deployment_params = json.dumps(dummy_user | json.loads(submission)) + deployment_params = dummy_user | json.loads(submission) env = { "PATH": settings.bin_path, + # used in nixos-anywhere for ssh-copy-id + "HOME": expanduser("~"), + } | { # pass in form info to our deployment - "DEPLOYMENT": deployment_params, + # FIXME: ensure sensitive info is protected + f"TF_VAR_{k}": v if isinstance(v, str) else json.dumps(v) for k, v in deployment_params.items() } + cwd = f"{settings.repo_dir}/launch" + # FIXME: move init to packaging phase cmd = [ - "nix", - "develop", - "--extra-experimental-features", - "configurable-impure-env", - "--command", - "nixops4", - "apply", - "test", + "tofu", + # f"-chdir={cwd}", + "init", + "-get=false", + "-input=false", + # "-plugin-dir=...", ] - deployment_result = subprocess.run( - cmd, - cwd=settings.repo_dir, - env=env, - ) - return deployment_result, json.loads(deployment_params) + subprocess.run(cmd, cwd=cwd, env=env) + cmd = [ + "tofu", + # f"-chdir={cwd}", + "apply", + f"-state={cwd}/terraform.tfstate", # FIXME: separate users' state + "--auto-approve", + ] + deployment_result = subprocess.run(cmd, cwd=cwd, env=env) + print(deployment_result) + return deployment_result, deployment_params