diff --git a/infra/machines/fedi201/fedipanel.nix b/infra/machines/fedi201/fedipanel.nix index 4f90c473..f9fc1554 100644 --- a/infra/machines/fedi201/fedipanel.nix +++ b/infra/machines/fedi201/fedipanel.nix @@ -19,6 +19,24 @@ in enable = true; production = true; domain = "demo.fediversity.eu"; + # FIXME: make it work without this duplication + settings = + let + cfg = config.services.${name}; + in + { + STATIC_ROOT = "/var/lib/${name}/static"; + DEBUG = false; + ALLOWED_HOSTS = [ + cfg.domain + cfg.host + "localhost" + "[::1]" + ]; + CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ]; + COMPRESS_OFFLINE = true; + LIBSASS_OUTPUT_STYLE = "compressed"; + }; secrets = { SECRET_KEY = config.age.secrets.panel-secret-key.path; }; diff --git a/launch/.envrc b/launch/.envrc new file mode 100644 index 00000000..26ef376b --- /dev/null +++ b/launch/.envrc @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +# shellcheck shell=bash +if type -P lorri &>/dev/null; then + eval "$(lorri direnv --flake .)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use flake +fi diff --git a/launch/.terraform.lock.hcl b/launch/.terraform.lock.hcl new file mode 100644 index 00000000..33548dcd --- /dev/null +++ b/launch/.terraform.lock.hcl @@ -0,0 +1,16 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/external" { + version = "2.3.4" + hashes = [ + "h1:HfVaWMC7Tz+tRfoWZtGCX2MATcgX3HsexoirWdi/voo=", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:qTlGDGC3RmXIPLgwsIh4LHG/DrAR6T6L+Wn6egnQnwE=", + ] +} diff --git a/launch/deploy.sh b/launch/deploy.sh new file mode 100755 index 00000000..efdc75b5 --- /dev/null +++ b/launch/deploy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -uex -o pipefail +declare domain hostname config initialUser +eval "$(jq -r '@sh "domain=\(.domain) hostname=\(.hostname) config=\(.config) initialUser=\(.initialUser)"')" +TARGET_HOST="${hostname}.abundos.eu" +TARGET="root@${TARGET_HOST}" +wrapper="$(mktemp -d)/wrapper.nix" +echo "(import $(readlink -f "./${config}.nix")).extendModules { specialArgs.terraform = { domain = \"${domain}\"; hostname = \"${hostname}\"; initialUser = builtins.fromJSON ''${initialUser}''; }; }" > "$wrapper" +NIXOS_SYSTEM=$(nix build --no-link --json --option show-trace true --file "$wrapper" "config.system.build.toplevel" | jq -r '.[].outputs.out') +sshOpts=(-p 22 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no) +NIX_SSHOPTS="${sshOpts[*]}" nix copy -s --experimental-features nix-command --to "ssh://$TARGET" "$NIXOS_SYSTEM" +switchCommand="nix-env -p /nix/var/nix/profiles/system --set $(printf "%q" "$NIXOS_SYSTEM"); /nix/var/nix/profiles/system/bin/switch-to-configuration switch" +deploy_status=0 +# shellcheck disable=SC2029 +ssh "${sshOpts[@]}" "$TARGET" "$switchCommand" || deploy_status="$?" +exit "$deploy_status" diff --git a/launch/env.nix b/launch/env.nix new file mode 100644 index 00000000..a517b475 --- /dev/null +++ b/launch/env.nix @@ -0,0 +1,11 @@ +{ + pkgs, + ... +}: +pkgs.stdenv.mkDerivation { + name = "fediversity-repo"; + src = ../.; + installPhase = '' + cp -r . $out + ''; +} diff --git a/launch/garage.nix b/launch/garage.nix new file mode 100644 index 00000000..b79614f4 --- /dev/null +++ b/launch/garage.nix @@ -0,0 +1,37 @@ +let + ## NOTE: All of these secrets are publicly available in this source file + ## and will end up in the Nix store. We don't care as they are only ever + ## used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +import ./shared.nix { + module = + { pkgs, ... }: + { + fediversity = { + garage.enable = true; + pixelfed = pixelfedS3KeyConfig { inherit pkgs; }; + mastodon = mastodonS3KeyConfig { inherit pkgs; }; + peertube = peertubeS3KeyConfig { inherit pkgs; }; + }; + }; +} diff --git a/launch/mastodon.nix b/launch/mastodon.nix new file mode 100644 index 00000000..def88630 --- /dev/null +++ b/launch/mastodon.nix @@ -0,0 +1,20 @@ +let + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; +in +import ./shared.nix { + module = + { pkgs, ... }: + { + fediversity = { + mastodon = mastodonS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + temp.cores = 1; # FIXME: should come from NixOps4 eventually + }; + }; +} diff --git a/launch/peertube.nix b/launch/peertube.nix new file mode 100644 index 00000000..df58c0a7 --- /dev/null +++ b/launch/peertube.nix @@ -0,0 +1,23 @@ +let + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; +in +import ./shared.nix { + module = + { pkgs, ... }: + { + fediversity = { + peertube = peertubeS3KeyConfig { inherit pkgs; } // { + enable = true; + ## NOTE: Only ever used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24"; + }; + }; + }; +} diff --git a/launch/pixelfed.nix b/launch/pixelfed.nix new file mode 100644 index 00000000..6fdaf65a --- /dev/null +++ b/launch/pixelfed.nix @@ -0,0 +1,19 @@ +let + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +import ./shared.nix { + module = + { pkgs, ... }: + { + fediversity = { + pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + }; + }; +} diff --git a/launch/resource.nix b/launch/resource.nix new file mode 100644 index 00000000..04811271 --- /dev/null +++ b/launch/resource.nix @@ -0,0 +1,41 @@ +{ + lib, + config, + ... +}: + +let + inherit (lib) attrValues elem mkDefault; + inherit (lib.attrsets) concatMapAttrs optionalAttrs; + inherit (lib.strings) removeSuffix; + + secretsPrefix = ../secrets; + secrets = import (secretsPrefix + "/secrets.nix"); + keys = import ../keys; + +in +{ + fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name}; + + ## The configuration of the machine. We strive to keep in this file only the + ## options that really need to be injected from the resource. Everything else + ## should go into the `./nixos` subdirectory. + imports = [ + ../infra/common/options.nix + ../infra/common/nixos + ]; + + ## Read all the secrets, filter the ones that are supposed to be readable + ## with this host's public key, and add them correctly to the configuration + ## as `age.secrets..file`. + age.secrets = concatMapAttrs ( + name: secret: + optionalAttrs (elem config.fediversityVm.hostPublicKey secret.publicKeys) { + ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}"; + } + ) secrets; + + ## FIXME: Remove direct root authentication once the NixOps4 NixOS provider + ## supports users with password-less sudo. + users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors; +} diff --git a/launch/shared.nix b/launch/shared.nix new file mode 100644 index 00000000..f3d18344 --- /dev/null +++ b/launch/shared.nix @@ -0,0 +1,45 @@ +{ + system ? "x86_64-linux", + sources ? import ../npins, + pkgs ? import sources.nixpkgs { + inherit system; + config = { }; + overlays = [ (import ../panel/nix/overlay.nix) ]; + }, + module, + ... +}: +import "${sources.nixpkgs}/nixos/lib/eval-config.nix" { + modules = [ + "${sources.disko}/module.nix" + "${sources.agenix}/modules/age.nix" + ../services/fediversity + ./resource.nix + # FIXME: get VM details from TF + module + ( + { + terraform, + ... + }: + let + inherit (terraform) hostname; + in + { + imports = [ + ../infra/test-machines/${hostname} + ]; + fediversityVm.name = hostname; + fediversity = { + inherit (terraform) domain; + temp.initialUser = { + inherit (terraform.initialUser) username email displayName; + # FIXME: disgusting, but nvm, this is going to be replaced by + # proper central authentication at some point + passwordFile = pkgs.writeText "password" terraform.initialUser.password; + }; + }; + } + ) + ]; +} diff --git a/npins/sources.json b/npins/sources.json index 45efd6b4..7e1df4f9 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -1,5 +1,44 @@ { "pins": { + "agenix": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "ryantm", + "repo": "agenix" + }, + "branch": "main", + "revision": "e600439ec4c273cf11e06fe4d9d906fb98fa097c", + "url": "https://github.com/ryantm/agenix/archive/e600439ec4c273cf11e06fe4d9d906fb98fa097c.tar.gz", + "hash": "006ngydiykjgqs85cl19h9klq8kaqm5zs0ng51dnwy7nzgqxzsdr" + }, + "disko": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "nix-community", + "repo": "disko" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "version": "v1.11.0", + "revision": "cdf8deded8813edfa6e65544f69fdd3a59fa2bb4", + "url": "https://api.github.com/repos/nix-community/disko/tarball/v1.11.0", + "hash": "13brimg7z7k9y36n4jc1pssqyw94nd8qvgfjv53z66lv4xkhin92" + }, + "flake-inputs": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "fricklerhandwerk", + "repo": "flake-inputs" + }, + "branch": "main", + "revision": "559574c9cbb8af262f3944b67d60fbf0f6ad03c3", + "url": "https://github.com/fricklerhandwerk/flake-inputs/archive/559574c9cbb8af262f3944b67d60fbf0f6ad03c3.tar.gz", + "hash": "0gbhmp6x2vdzvfnsvqzal3g8f8hx2ia6r73aibc78kazf78m67x6" + }, "htmx": { "type": "GitRelease", "repository": { @@ -27,12 +66,24 @@ "url": "https://github.com/nix-community/nix-unit/archive/2071bbb765681ac3d8194ec560c8b27ff2a3b541.tar.gz", "hash": "0blz1kcmn9vnr9q3iqp2mv13hv3pdccljmmc54f8j7ybf5v0wgmp" }, + "nixos-anywhere": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "KiaraGrouwstra", + "repo": "nixos-anywhere" + }, + "branch": "special-args-nested-flake-fixed", + "revision": "5aa35145f045eb23fa8773821d5626bcf54dbe0e", + "url": "https://github.com/KiaraGrouwstra/nixos-anywhere/archive/5aa35145f045eb23fa8773821d5626bcf54dbe0e.tar.gz", + "hash": "0m67iyd04wl183il1cfi623xpxcvbbpc5x1gh74478qc3fgr0g54" + }, "nixpkgs": { "type": "Channel", "name": "nixpkgs-unstable", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre711046.8edf06bea5bc/nixexprs.tar.xz", - "hash": "1mwsn0rvfm603svrq3pca4c51zlix5gkyr4gl6pxhhq3q6xs5s8y" + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre777917.b7ba7f9f45c5/nixexprs.tar.xz", + "hash": "0jb6b7sv66bn06pchj2l88z0i5dlz0c2vb3z6pjjlq2p8q11zigg" } }, "version": 3 -} \ No newline at end of file +} diff --git a/panel/default.nix b/panel/default.nix index 767802be..06d1a487 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -20,11 +20,18 @@ in pkgs.npins manage ]; - env = import ./env.nix { inherit lib pkgs; } // { - NPINS_DIRECTORY = toString ../npins; - CREDENTIALS_DIRECTORY = toString ./.credentials; - DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; - }; + env = + let + inherit (builtins) toString; + in + import ./env.nix { inherit lib pkgs; } + // { + NPINS_DIRECTORY = toString ../npins; + CREDENTIALS_DIRECTORY = toString ./.credentials; + DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; + # locally: use a fixed relative reference, so we can use our newest files without copying to the store + REPO_DIR = toString ../.; + }; shellHook = '' ln -sf ${sources.htmx}/dist/htmx.js src/panel/static/htmx.min.js # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd. diff --git a/panel/env.nix b/panel/env.nix index 07ce4193..ad184cad 100644 --- a/panel/env.nix +++ b/panel/env.nix @@ -3,16 +3,13 @@ pkgs, ... }: -let - inherit (builtins) toString; -in { - REPO_DIR = toString ../.; - # explicitly use nix, as e.g. lix does not have configurable-impure-env BIN_PATH = lib.makeBinPath [ - # explicitly use nix, as e.g. lix does not have configurable-impure-env - pkgs.nix - # nixops error maybe due to our flake git hook: executing 'git': No such file or directory + pkgs.lix + pkgs.bash + pkgs.coreutils + pkgs.openssh pkgs.git + pkgs.jq # implicit dep of nixos-anywhere TF: https://github.com/nix-community/nixos-anywhere/issues/416 ]; } diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index 27359503..01d90df6 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -29,6 +29,7 @@ let ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) (builtins.toFile "extra-settings.py" cfg.extra-settings) ]; + REPO_DIR = import ../../launch/env.nix { inherit lib pkgs; }; }; python-environment = pkgs.python3.withPackages ( diff --git a/panel/nix/package.nix b/panel/nix/package.nix index 9337887c..cd5a691a 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -1,5 +1,6 @@ { lib, + pkgs, sqlite, python3, sources ? import ../../npins, @@ -11,7 +12,7 @@ let root = ../src; fileset = intersection (gitTracked ../../.) ../src; }; - pyproject = with lib; fromTOML pyproject-toml; + pyproject = fromTOML pyproject-toml; # TODO: define this globally name = "panel"; # TODO: we may want this in a file so it's easier to read statically @@ -58,7 +59,9 @@ python3.pkgs.buildPythonPackage { mkdir -p $out/bin cp -v ${src}/manage.py $out/bin/manage.py chmod +x $out/bin/manage.py - wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH" + wrapProgram $out/bin/manage.py \ + --set REPO_DIR "${import ../../launch/env.nix { inherit lib pkgs; }}" \ + --prefix PYTHONPATH : "$PYTHONPATH" cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/ ''; } diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index b270612c..ccdb8308 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -10,7 +10,9 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/4.2/ref/settings/ """ +import re import sys +import subprocess import os import importlib.util import dj_database_url @@ -18,6 +20,8 @@ import dj_database_url from os import environ as env from pathlib import Path +STORE_PATTERN = re.compile("^/nix/store/[^/]+$") + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -171,6 +175,54 @@ COMPRESS_PRECOMPILERS = [ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", + }, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", + }, + }, + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", + } + }, + "handlers": { + "console": { + "level": "INFO", + # "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + }, + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", + }, + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, + }, + "loggers": { + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", + }, + "django.server": { + "handlers": ["django.server"], + "level": "INFO", + "propagate": False, + }, + }, +} + # Customization via user settings # This must be at the end, as it must be able to override the above # TODO(@fricklerhandwerk): diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 5c423fc6..1aced5a8 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -1,5 +1,6 @@ from enum import Enum import json +import logging import subprocess import os @@ -13,6 +14,8 @@ from django.shortcuts import render from panel import models, settings from panel.configuration import forms +logger = logging.getLogger(__name__) + class Index(TemplateView): template_name = 'index.html' @@ -102,7 +105,7 @@ class DeploymentStatus(ConfigurationForm): # Check for deploy button if "deploy" in self.request.POST.keys(): deployment_result, deployment_params = self.deployment(obj) - deployment_succeeded = deployment_result.returncode == 0 + deployment_succeeded = deployment_result == 0 return render(self.request, "partials/deployment_result.html", { "deployment_succeeded": deployment_succeeded, @@ -123,36 +126,62 @@ class DeploymentStatus(ConfigurationForm): }) def deployment(self, obj): - submission = obj.parsed_value.model_dump_json() + testvms = { + "garage": "test01", + "mastodon": "test06", + "pixelfed": "test04", + "peertube": "test03", + } + submission = json.loads(obj.parsed_value.model_dump_json()) + logger.debug(f"submission: {submission}") # FIXME: let the user specify these from the form (#190) dummy_user = { - "initialUser": { + "initialUser": json.dumps({ "displayName": "Testy McTestface", "username": "test", "password": "testtest", "email": "test@test.com", - }, + }), } - # serialize back and forth now we still need to manually inject the dummy user - deployment_params = json.dumps(dummy_user | json.loads(submission)) - env = { - "PATH": settings.bin_path, - # pass in form info to our deployment - "DEPLOYMENT": deployment_params, - } - cmd = [ - "nix", - "develop", - "--extra-experimental-features", - "configurable-impure-env", - "--command", - "nixops4", - "apply", - "test", - ] - deployment_result = subprocess.run( - cmd, - cwd=settings.repo_dir, - env=env, - ) + deployment_result = 0 + if submission["enable"]: + services = [ + "mastodon", + "pixelfed", + "peertube", + ] + for service in [ + "garage", + ] + services: + if (service == "garage" and any(submission[srvc]["enable"] for srvc in services)) or submission[service]["enable"]: + hostname = testvms[service] + # serialize back and forth now we still need to manually inject the dummy user + deployment_params = json.dumps({ + "config": service, + "hostname": hostname, + "initialUser": json.dumps({ + "displayName": "Testy McTestface", + "username": "test", + "password": "testtest", + "email": "test@test.com", + }), + } | submission) + logger.debug(f"deployment_params: {deployment_params}") + deployment_result = deployment_result or subprocess.run( + ["./deploy.sh"], + cwd=f"{settings.repo_dir}/launch", + env={ + "PATH": settings.bin_path, + # warning: ignoring error initializing Lix fetcher cache: error: creating directory '/var/empty/.cache': Operation not permitted + "HOME": "/tmp", + }, + # pass in form info to our deployment + input=deployment_params, + text=True, + ).returncode + else: + logger.debug(f"service {service} disabled") + else: + # FIXME: implement disable + logger.debug("deployment disabled") return deployment_result, json.loads(deployment_params)