1
0
Fork 0
This commit is contained in:
kiara Grouwstra 2025-03-24 10:41:07 +01:00
parent 88674c8efc
commit 6bb5768ddc
Signed by: kiara
SSH key fingerprint: SHA256:COspvLoLJ5WC5rFb9ZDe5urVCkK4LJZOsjfF4duRJFU
22 changed files with 693 additions and 23 deletions

View file

@ -19,6 +19,24 @@ in
enable = true;
production = true;
domain = "demo.fediversity.eu";
# FIXME: make it work without this duplication
settings =
let
cfg = config.services.${name};
in
{
STATIC_ROOT = "/var/lib/${name}/static";
DEBUG = false;
ALLOWED_HOSTS = [
cfg.domain
cfg.host
"localhost"
"[::1]"
];
CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ];
COMPRESS_OFFLINE = true;
LIBSASS_OUTPUT_STYLE = "compressed";
};
secrets = {
SECRET_KEY = config.age.secrets.panel-secret-key.path;
};

1
launch/.auto.tfvars.json Normal file
View file

@ -0,0 +1 @@
{"nixos-anywhere": "/nix/store/y5ripxky2azz90fzpsj6383f1vljrxkg-source"}

10
launch/.envrc Normal file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# the shebang is ignored, but nice for editors
# shellcheck shell=bash
if type -P lorri &>/dev/null; then
eval "$(lorri direnv --flake .)"
else
echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
use flake
fi

3
launch/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
.terraform/
.terraform.tfstate.lock.info
terraform.tfstate*

16
launch/.terraform.lock.hcl generated Normal file
View file

@ -0,0 +1,16 @@
# This file is maintained automatically by "tofu init".
# Manual edits may be lost in future updates.
provider "registry.opentofu.org/hashicorp/external" {
version = "2.3.4"
hashes = [
"h1:F45RHS5NlHBWkfcW61rwkA7Z2P+p3Rxvq75XMhNeFHE=",
]
}
provider "registry.opentofu.org/hashicorp/null" {
version = "3.2.3"
hashes = [
"h1:P9gA/YsRU4Q+STK6EkCOjTHYUop/+1LfIbX7QMNHfPk=",
]
}

@ -0,0 +1 @@
Subproject commit 657cc08c2a22cf6500de70df5519ccb194449f0f

@ -0,0 +1 @@
Subproject commit 657cc08c2a22cf6500de70df5519ccb194449f0f

9
launch/README.md Normal file
View file

@ -0,0 +1,9 @@
# service deployment
## usage
### updating TF modules
```sh
echo "{\"nixos-anywhere\": $(nix-instantiate --eval --json -E '(import ../npins).nixos-anywhere.outPath')}" > .auto.tfvars.json
```

22
launch/default.nix Normal file
View file

@ -0,0 +1,22 @@
{
system ? builtins.currentSystem,
sources ? import ../npins,
pkgs ? import sources.nixpkgs {
inherit system;
},
}@args:
let
inherit (pkgs) lib;
in
{
shell = pkgs.mkShellNoCC {
packages = [
pkgs.npins
pkgs.jq # implicit dep of nixos-anywhere TF: https://github.com/nix-community/nixos-anywhere/issues/416
(import ./tf.nix { inherit lib pkgs; })
];
};
}
# re-export inputs so they can be overridden granularly
# (they can't be accessed from the outside any other way)
// args

158
launch/flake.lock generated Normal file
View file

@ -0,0 +1,158 @@
{
"nodes": {
"agenix": {
"inputs": {
"darwin": "darwin",
"home-manager": "home-manager",
"nixpkgs": "nixpkgs",
"systems": "systems"
},
"locked": {
"lastModified": 1736955230,
"narHash": "sha256-uenf8fv2eG5bKM8C/UvFaiJMZ4IpUFaQxk9OH5t/1gA=",
"owner": "ryantm",
"repo": "agenix",
"rev": "e600439ec4c273cf11e06fe4d9d906fb98fa097c",
"type": "github"
},
"original": {
"owner": "ryantm",
"repo": "agenix",
"type": "github"
}
},
"darwin": {
"inputs": {
"nixpkgs": [
"agenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1700795494,
"narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
"owner": "lnl7",
"repo": "nix-darwin",
"rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
"type": "github"
},
"original": {
"owner": "lnl7",
"ref": "master",
"repo": "nix-darwin",
"type": "github"
}
},
"disko": {
"inputs": {
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1741786315,
"narHash": "sha256-VT65AE2syHVj6v/DGB496bqBnu1PXrrzwlw07/Zpllc=",
"owner": "nix-community",
"repo": "disko",
"rev": "0d8c6ad4a43906d14abd5c60e0ffe7b587b213de",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "disko",
"type": "github"
}
},
"home-manager": {
"inputs": {
"nixpkgs": [
"agenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1703113217,
"narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "home-manager",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1703013332,
"narHash": "sha256-+tFNwMvlXLbJZXiMHqYq77z/RfmpfpiI3yjL6o/Zo9M=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "54aac082a4d9bb5bbc5c4e899603abfb76a3f6d6",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1741402956,
"narHash": "sha256-y2hByvBM03s9T2fpeLjW6iprbxnhV9mJMmSwCHc41ZQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ed0b1881565c1ffef490c10d663d4f542031dad3",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_3": {
"locked": {
"lastModified": 1742753982,
"narHash": "sha256-WYryX6lCrmh4AaEoBHA1zwTTs6vPtevT3/ywGyzz4SI=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "ec9c54e7a9feec999aa34a15781080bed5082306",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "release-24.11",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"agenix": "agenix",
"disko": "disko",
"nixpkgs": "nixpkgs_3"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

125
launch/flake.nix Normal file
View file

@ -0,0 +1,125 @@
{
inputs = {
agenix.url = "github:ryantm/agenix";
disko.url = "github:nix-community/disko";
nixpkgs.url = "github:nixos/nixpkgs/release-24.11";
};
outputs =
inputs@{ nixpkgs, ... }:
let
system = "x86_64-linux";
inherit (nixpkgs) lib;
in
{
nixosConfigurations =
let
## NOTE: All of these secrets are publicly available in this source file
## and will end up in the Nix store. We don't care as they are only ever
## used for testing anyway.
##
## FIXME: Generate and store in NixOps4's state.
mastodonS3KeyConfig =
{ pkgs, ... }:
{
s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558";
s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34";
};
peertubeS3KeyConfig =
{ pkgs, ... }:
{
s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b";
s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395";
};
pixelfedS3KeyConfig =
{ pkgs, ... }:
{
s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b";
s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987";
};
in
lib.mapAttrs
(
_: module:
lib.nixosSystem {
inherit system;
specialArgs = { inherit system inputs; };
modules = [
inputs.disko.nixosModules.default
inputs.agenix.nixosModules.default
../services/fediversity
./resource.nix
module
{
nixpkgs = { inherit system; };
}
(
{ pkgs, terraform, ... }:
let
inherit (terraform) hostname;
in
{
imports = [
# FIXME: get VM details from TF
../infra/test-machines/${hostname}
];
fediversityVm.name = hostname;
fediversity = {
inherit (terraform) domain;
temp.initialUser = {
inherit (terraform.initialUser) username email displayName;
# FIXME: disgusting, but nvm, this is going to be replaced by
# proper central authentication at some point
passwordFile = pkgs.writeText "password" terraform.initialUser.password;
};
};
}
)
];
}
)
{
garage =
{ pkgs, ... }:
{
fediversity = {
garage.enable = true;
pixelfed = pixelfedS3KeyConfig { inherit pkgs; };
mastodon = mastodonS3KeyConfig { inherit pkgs; };
peertube = peertubeS3KeyConfig { inherit pkgs; };
};
};
mastodon =
{ pkgs, ... }:
{
fediversity = {
mastodon = mastodonS3KeyConfig { inherit pkgs; } // {
enable = true;
};
temp.cores = 1; # FIXME: should come from NixOps4 eventually
};
};
peertube =
{ pkgs, ... }:
{
fediversity = {
peertube = peertubeS3KeyConfig { inherit pkgs; } // {
enable = true;
## NOTE: Only ever used for testing anyway.
##
## FIXME: Generate and store in NixOps4's state.
secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24";
};
};
};
pixelfed =
{ pkgs, ... }:
{
fediversity = {
pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // {
enable = true;
};
};
};
};
};
}

84
launch/main.tf Normal file
View file

@ -0,0 +1,84 @@
variable "domain" {
type = string
default = "fediversity.net"
}
variable "mastodon" {
type = object({
enable = bool
})
default = {
enable = false
}
}
variable "pixelfed" {
type = object({
enable = bool
})
default = {
enable = false
}
}
variable "peertube" {
type = object({
enable = bool
})
default = {
enable = false
}
}
variable "initialUser" {
type = object({
displayName = string
username = string
email = string
# TODO: mark (nested) credentials as sensitive
# https://discuss.hashicorp.com/t/is-it-possible-to-mark-an-attribute-of-an-object-as-sensitive/24649/2
password = string
})
default = {
displayName = "Testy McTestface"
username = "test"
email = "test@test.com"
password = "testtest"
}
}
# module "garage" {
# source = "./vm"
# count = var.mastodon.enable || var.pixelfed.enable || var.peertube.enable ? 1 : 0
# domain = var.domain
# hostname = "test01"
# config = "garage"
# initialUser = var.initialUser
# }
module "mastodon" {
source = "./vm"
count = var.mastodon.enable ? 1 : 0
domain = var.domain
hostname = "test02"
config = "mastodon"
initialUser = var.initialUser
}
module "pixelfed" {
source = "./vm"
count = var.pixelfed.enable ? 1 : 0
domain = var.domain
hostname = "test04"
config = "pixelfed"
initialUser = var.initialUser
}
module "peertube" {
source = "./vm"
count = var.peertube.enable ? 1 : 0
domain = var.domain
hostname = "test03"
config = "peertube"
initialUser = var.initialUser
}

15
launch/pass-ssh-key.sh Executable file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env bash
export host="$host"
mkdir -p etc/ssh
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
for keyname in ssh_host_ed25519_key ssh_host_ed25519_key.pub; do
if [[ $keyname == *.pub ]]; then
umask 0133
else
umask 0177
fi
cp "$SCRIPT_DIR/../infra/test-machines/${host}/${keyname}" ./etc/ssh/${keyname}
done

41
launch/resource.nix Normal file
View file

@ -0,0 +1,41 @@
{
lib,
config,
...
}:
let
inherit (lib) attrValues elem mkDefault;
inherit (lib.attrsets) concatMapAttrs optionalAttrs;
inherit (lib.strings) removeSuffix;
secretsPrefix = ../secrets;
secrets = import (secretsPrefix + "/secrets.nix");
keys = import ../keys;
in
{
fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name};
## The configuration of the machine. We strive to keep in this file only the
## options that really need to be injected from the resource. Everything else
## should go into the `./nixos` subdirectory.
imports = [
../infra/common/options.nix
../infra/common/nixos
];
## Read all the secrets, filter the ones that are supposed to be readable
## with this host's public key, and add them correctly to the configuration
## as `age.secrets.<name>.file`.
age.secrets = concatMapAttrs (
name: secret:
optionalAttrs (elem config.fediversityVm.hostPublicKey secret.publicKeys) {
${removeSuffix ".age" name}.file = secretsPrefix + "/${name}";
}
) secrets;
## FIXME: Remove direct root authentication once the NixOps4 NixOS provider
## supports users with password-less sudo.
users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors;
}

1
launch/shell.nix Normal file
View file

@ -0,0 +1 @@
(import ./. { }).shell

25
launch/tf.nix Normal file
View file

@ -0,0 +1,25 @@
# FIXME: use overlays so this gets imported just once?
{
lib,
pkgs,
...
}:
let
tofuProvider =
provider:
provider.override (oldArgs: {
provider-source-address =
lib.replaceStrings [ "https://registry.terraform.io/providers" ] [ "registry.opentofu.org" ]
oldArgs.homepage;
});
tf = pkgs.opentofu;
tfPlugins = (
p: [
p.null
p.external
]
);
in
# tf.withPlugins tfPlugins
# https://github.com/NixOS/nixpkgs/pull/358522
tf.withPlugins (p: pkgs.lib.lists.map tofuProvider (tfPlugins p))

46
launch/vm/main.tf Normal file
View file

@ -0,0 +1,46 @@
variable "domain" {
type = string
}
variable "hostname" {
type = string
}
variable "config" {
type = string
}
variable "initialUser" {
type = object({
displayName = string
username = string
password = string
email = string
})
}
module "deploy" {
# source = "github.com/nix-community/nixos-anywhere//terraform/all-in-one"
source = "github.com/KiaraGrouwstra/nixos-anywhere?ref=special-args-nested-flake-fixed//terraform/all-in-one"
nixos_system_attr = ".#nixosConfigurations.${var.config}.config.system.build.toplevel"
nixos_partitioner_attr = ".#nixosConfigurations.${var.config}.config.system.build.diskoScriptNoDeps"
# when instance id changes, it will trigger a reinstall
instance_id = var.hostname
target_user = "kiara"
target_host = "${var.hostname}.abundos.eu"
extra_files_script = "${path.module}/../pass-ssh-key.sh"
extra_environment = {
host = var.hostname
}
special_args = {
terraform = {
domain = var.domain
hostname = var.hostname
initialUser = var.initialUser
}
}
nix_options = {
show-trace = true
}
# build_on_remote = true
}

View file

@ -27,6 +27,18 @@
"url": "https://github.com/nix-community/nix-unit/archive/2071bbb765681ac3d8194ec560c8b27ff2a3b541.tar.gz",
"hash": "0blz1kcmn9vnr9q3iqp2mv13hv3pdccljmmc54f8j7ybf5v0wgmp"
},
"nixos-anywhere": {
"type": "Git",
"repository": {
"type": "GitHub",
"owner": "KiaraGrouwstra",
"repo": "nixos-anywhere"
},
"branch": "special-args-nested-flake-fixed",
"revision": "bbf54831f177b4ae52b55758ac725583617c39f9",
"url": "https://github.com/KiaraGrouwstra/nixos-anywhere/archive/bbf54831f177b4ae52b55758ac725583617c39f9.tar.gz",
"hash": "11m8r632yp4s8sy106zk7a1inp44gvwq6hinjf2pczpr6a5hxanx"
},
"nixpkgs": {
"type": "Channel",
"name": "nixpkgs-unstable",

View file

@ -7,12 +7,15 @@ let
inherit (builtins) toString;
in
{
# locally: use a fixed relative reference, so we can use our newest files without copying to the store
REPO_DIR = toString ../.;
# explicitly use nix, as e.g. lix does not have configurable-impure-env
BIN_PATH = lib.makeBinPath [
# explicitly use nix, as e.g. lix does not have configurable-impure-env
pkgs.nix
# nixops error maybe due to our flake git hook: executing 'git': No such file or directory
pkgs.lix
pkgs.bash
pkgs.coreutils
pkgs.openssh
pkgs.git
pkgs.jq # implicit dep of nixos-anywhere TF: https://github.com/nix-community/nixos-anywhere/issues/416
(import ../launch/tf.nix { inherit lib pkgs; })
];
}

View file

@ -11,7 +11,7 @@ let
root = ../src;
fileset = intersection (gitTracked ../../.) ../src;
};
pyproject = with lib; fromTOML pyproject-toml;
pyproject = fromTOML pyproject-toml;
# TODO: define this globally
name = "panel";
# TODO: we may want this in a file so it's easier to read statically
@ -58,7 +58,9 @@ python3.pkgs.buildPythonPackage {
mkdir -p $out/bin
cp -v ${src}/manage.py $out/bin/manage.py
chmod +x $out/bin/manage.py
wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
wrapProgram $out/bin/manage.py \
--set REPO_DIR "${../..}" \
--prefix PYTHONPATH : "$PYTHONPATH"
cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/
'';
}

View file

@ -10,7 +10,9 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
import re
import sys
import subprocess
import os
import importlib.util
import dj_database_url
@ -18,6 +20,8 @@ import dj_database_url
from os import environ as env
from pathlib import Path
STORE_PATTERN = re.compile("^/nix/store/[^/]+$")
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
@ -171,6 +175,54 @@ COMPRESS_PRECOMPILERS = [
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"require_debug_false": {
"()": "django.utils.log.RequireDebugFalse",
},
"require_debug_true": {
"()": "django.utils.log.RequireDebugTrue",
},
},
"formatters": {
"django.server": {
"()": "django.utils.log.ServerFormatter",
"format": "[{server_time}] {message}",
"style": "{",
}
},
"handlers": {
"console": {
"level": "INFO",
# "filters": ["require_debug_true"],
"class": "logging.StreamHandler",
},
"django.server": {
"level": "INFO",
"class": "logging.StreamHandler",
"formatter": "django.server",
},
"mail_admins": {
"level": "ERROR",
"filters": ["require_debug_false"],
"class": "django.utils.log.AdminEmailHandler",
},
},
"loggers": {
"django": {
"handlers": ["console", "mail_admins"],
"level": "INFO",
},
"django.server": {
"handlers": ["django.server"],
"level": "INFO",
"propagate": False,
},
},
}
# Customization via user settings
# This must be at the end, as it must be able to override the above
# TODO(@fricklerhandwerk):
@ -186,6 +238,21 @@ if user_settings_file is not None:
sys.modules["user_settings"] = module
from user_settings import * # noqa: F403 # pyright: ignore [reportMissingImports]
def handle_double_hash(orig_path: str) -> str:
"""Convert store paths to account for potentially double hashes.
c.f. https://github.com/NixOS/nix/issues/10627
"""
# local paths can stay as-is
if not STORE_PATTERN.match(orig_path):
return orig_path
# for /nix/store paths, account for double hashing
orig_name = orig_path.split("/")[-1]
cmd = ["find", "/nix/store/", "-maxdepth", "1", "-name", f"*{orig_name}"]
process = subprocess.run(cmd, capture_output=True)
fixed_name = process.stdout.decode("utf-8").split("\n")[0]
return fixed_name
# non-Django application settings
# TODO(@fricklerhandwerk):
@ -196,4 +263,4 @@ if user_settings_file is not None:
bin_path=env['BIN_PATH']
# path of the root flake to trigger nixops from, see #94.
# to deploy this should be specified, for dev just use a relative path.
repo_dir = env["REPO_DIR"]
repo_dir = handle_double_hash(env["REPO_DIR"])

View file

@ -1,5 +1,6 @@
from enum import Enum
import json
from os.path import expanduser
import subprocess
import os
@ -128,25 +129,34 @@ class DeploymentStatus(ConfigurationForm):
},
}
# serialize back and forth now we still need to manually inject the dummy user
deployment_params = json.dumps(dummy_user | json.loads(submission))
deployment_params = dummy_user | json.loads(submission)
env = {
"PATH": settings.bin_path,
# used in nixos-anywhere for ssh-copy-id
"HOME": expanduser("~"),
} | {
# pass in form info to our deployment
"DEPLOYMENT": deployment_params,
# FIXME: ensure sensitive info is protected
f"TF_VAR_{k}": v if isinstance(v, str) else json.dumps(v) for k, v in deployment_params.items()
}
cwd = f"{settings.repo_dir}/launch"
# FIXME: move init to packaging phase
cmd = [
"nix",
"develop",
"--extra-experimental-features",
"configurable-impure-env",
"--command",
"nixops4",
"apply",
"test",
"tofu",
# f"-chdir={cwd}",
"init",
"-get=false",
"-input=false",
# "-plugin-dir=...",
]
deployment_result = subprocess.run(
cmd,
cwd=settings.repo_dir,
env=env,
)
return deployment_result, json.loads(deployment_params)
subprocess.run(cmd, cwd=cwd, env=env)
cmd = [
"tofu",
# f"-chdir={cwd}",
"apply",
f"-state={cwd}/terraform.tfstate", # FIXME: separate users' state
"--auto-approve",
]
deployment_result = subprocess.run(cmd, cwd=cwd, env=env)
print(deployment_result)
return deployment_result, deployment_params