From e07c6e99728a6f1a36b2516b8c0723d99c5d6cdc Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Sun, 13 Apr 2025 22:26:35 +0200 Subject: [PATCH] buttons works deployed --- flake.lock | 21 ++++ flake.nix | 7 +- infra/common/nixos/default.nix | 13 +- infra/common/resource.nix | 1 + infra/machines/fedi201/fedipanel.nix | 18 +++ launch/.envrc | 10 ++ launch/.gitignore | 5 + launch/.terraform.lock.hcl | 16 +++ launch/README.md | 24 ++++ launch/default.nix | 31 +++++ launch/garage.nix | 34 +++++ launch/main.tf | 179 +++++++++++++++++++++++++++ launch/mastodon.nix | 17 +++ launch/options.nix | 53 ++++++++ launch/pass-ssh-key.sh | 15 +++ launch/peertube.nix | 20 +++ launch/pixelfed.nix | 16 +++ launch/resource.nix | 44 +++++++ launch/shared.nix | 26 ++++ launch/shell.nix | 1 + launch/tf-env.nix | 26 ++++ launch/tf.nix | 25 ++++ npins/sources.json | 43 ++++++- panel/default.nix | 17 ++- panel/env.nix | 15 ++- panel/nix/configuration.nix | 8 +- panel/nix/package.nix | 7 +- panel/src/panel/settings.py | 61 +++++++++ panel/src/panel/views.py | 37 +++--- 29 files changed, 750 insertions(+), 40 deletions(-) create mode 100644 launch/.envrc create mode 100644 launch/.gitignore create mode 100644 launch/.terraform.lock.hcl create mode 100644 launch/README.md create mode 100644 launch/default.nix create mode 100644 launch/garage.nix create mode 100644 launch/main.tf create mode 100644 launch/mastodon.nix create mode 100644 launch/options.nix create mode 100755 launch/pass-ssh-key.sh create mode 100644 launch/peertube.nix create mode 100644 launch/pixelfed.nix create mode 100644 launch/resource.nix create mode 100644 launch/shared.nix create mode 100644 launch/shell.nix create mode 100644 launch/tf-env.nix create mode 100644 launch/tf.nix diff --git a/flake.lock b/flake.lock index ad891639..e955e13c 100644 --- a/flake.lock +++ b/flake.lock @@ -571,6 +571,26 @@ "type": "github" } }, + "home-manager_2": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1743860185, + "narHash": "sha256-TkhfJ+vH+iGxLQL6RJLObMmldAQpysVJ+p1WnnKyIeQ=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "b5e29565131802cc8adee7dccede794226da8614", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, "mk-naked-shell": { "flake": false, "locked": { @@ -1215,6 +1235,7 @@ "disko": "disko", "flake-parts": "flake-parts", "git-hooks": "git-hooks", + "home-manager": "home-manager_2", "nixops4": "nixops4", "nixops4-nixos": "nixops4-nixos", "nixpkgs": "nixpkgs_7" diff --git a/flake.nix b/flake.nix index 9e0a719b..377448b3 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,8 @@ nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; flake-parts.url = "github:hercules-ci/flake-parts"; git-hooks.url = "github:cachix/git-hooks.nix"; + home-manager.url = "github:nix-community/home-manager"; + home-manager.inputs.nixpkgs.follows = "nixpkgs"; agenix.url = "github:ryantm/agenix"; disko.url = "github:nix-community/disko"; @@ -43,7 +45,10 @@ pre-commit.settings.hooks = let ## Add a directory here if pre-commit hooks shouldn't apply to it. - optout = [ "npins" ]; + optout = [ + "npins" + "launch/.terraform" + ]; excludes = map (dir: "^${dir}/") optout; addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); in diff --git a/infra/common/nixos/default.nix b/infra/common/nixos/default.nix index b870ab03..c789a04e 100644 --- a/infra/common/nixos/default.nix +++ b/infra/common/nixos/default.nix @@ -1,8 +1,8 @@ -{ lib, ... }: +{ lib, pkgs, ... }: let inherit (lib) mkDefault; - + nixPath = "/run/current-system/nixpkgs"; in { imports = [ @@ -16,6 +16,15 @@ in system.stateVersion = "24.05"; # do not change nixpkgs.hostPlatform = mkDefault "x86_64-linux"; + # use flake's nixpkgs over channels + nix.nixPath = [ "nixpkgs=${nixPath}" ]; + system.extraSystemBuilderCmds = '' + ln -sv ${pkgs.path} $out/nixpkgs + ''; + systemd.tmpfiles.rules = [ + "L+ ${nixPath} - - - - ${pkgs.path}" + ]; + ## This is just nice to have, but it is also particularly important for the ## Forgejo CI runners because the Nix configuration in the actions is directly ## taken from here. diff --git a/infra/common/resource.nix b/infra/common/resource.nix index 4606ddf4..bc17e743 100644 --- a/infra/common/resource.nix +++ b/infra/common/resource.nix @@ -34,6 +34,7 @@ in imports = [ inputs.agenix.nixosModules.default inputs.disko.nixosModules.default + inputs.home-manager.nixosModules.home-manager ./options.nix ./nixos ]; diff --git a/infra/machines/fedi201/fedipanel.nix b/infra/machines/fedi201/fedipanel.nix index 5c4236fc..b49471bb 100644 --- a/infra/machines/fedi201/fedipanel.nix +++ b/infra/machines/fedi201/fedipanel.nix @@ -37,6 +37,24 @@ in enable = true; production = true; domain = "demo.fediversity.eu"; + # FIXME: make it work without this duplication + settings = + let + cfg = config.services.${name}; + in + { + STATIC_ROOT = "/var/lib/${name}/static"; + DEBUG = false; + ALLOWED_HOSTS = [ + cfg.domain + cfg.host + "localhost" + "[::1]" + ]; + CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ]; + COMPRESS_OFFLINE = true; + LIBSASS_OUTPUT_STYLE = "compressed"; + }; secrets = { SECRET_KEY = config.age.secrets.panel-secret-key.path; }; diff --git a/launch/.envrc b/launch/.envrc new file mode 100644 index 00000000..26ef376b --- /dev/null +++ b/launch/.envrc @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +# shellcheck shell=bash +if type -P lorri &>/dev/null; then + eval "$(lorri direnv --flake .)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use flake +fi diff --git a/launch/.gitignore b/launch/.gitignore new file mode 100644 index 00000000..5ab952ce --- /dev/null +++ b/launch/.gitignore @@ -0,0 +1,5 @@ +.auto.tfvars.json +.npins.json +.terraform/ +.terraform.tfstate.lock.info +terraform.tfstate* diff --git a/launch/.terraform.lock.hcl b/launch/.terraform.lock.hcl new file mode 100644 index 00000000..33548dcd --- /dev/null +++ b/launch/.terraform.lock.hcl @@ -0,0 +1,16 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/external" { + version = "2.3.4" + hashes = [ + "h1:HfVaWMC7Tz+tRfoWZtGCX2MATcgX3HsexoirWdi/voo=", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:qTlGDGC3RmXIPLgwsIh4LHG/DrAR6T6L+Wn6egnQnwE=", + ] +} diff --git a/launch/README.md b/launch/README.md new file mode 100644 index 00000000..4c24dae6 --- /dev/null +++ b/launch/README.md @@ -0,0 +1,24 @@ +# service deployment + +## usage + +<-- TODO: port to just --> + +### updating npins + +```sh +$ cd launch/ +$ echo "$(nix eval --json -f ../npins)" > .npins.json +``` + +### local development + +```sh +$ nix-shell +$ eval "$(ssh-agent -s)" +# set your ssh key, e.g.: +$ ssh_key="$(readlink -f ~/.ssh/id_ed25519)" +$ echo "{\"ssh_private_key_file\": \"${ssh_key}\", \"deploy_environment\": {\"SSH_AUTH_SOCK\": \"${SSH_AUTH_SOCK}\"}}" > .auto.tfvars.json +$ rm -rf .terraform/ +$ tofu init +``` diff --git a/launch/default.nix b/launch/default.nix new file mode 100644 index 00000000..bb96db72 --- /dev/null +++ b/launch/default.nix @@ -0,0 +1,31 @@ +{ + system ? builtins.currentSystem, + sources ? import ../npins, + inputs ? import sources.flake-inputs { + root = ../.; + }, + # match the same version of opentofu that is deployed by the root flake + pkgs ? import inputs.nixpkgs { + inherit system; + }, +}: +let + inherit (pkgs) lib; +in +{ + shell = pkgs.mkShellNoCC { + packages = [ + pkgs.npins + pkgs.jaq # tf + (import ./tf.nix { inherit lib pkgs; }) + ]; + }; + + # re-export inputs so they can be overridden granularly + # (they can't be accessed from the outside any other way) + inherit + sources + system + pkgs + ; +} diff --git a/launch/garage.nix b/launch/garage.nix new file mode 100644 index 00000000..559ca37f --- /dev/null +++ b/launch/garage.nix @@ -0,0 +1,34 @@ +{ pkgs, ... }: +let + ## NOTE: All of these secrets are publicly available in this source file + ## and will end up in the Nix store. We don't care as they are only ever + ## used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +{ + fediversity = { + garage.enable = true; + pixelfed = pixelfedS3KeyConfig { inherit pkgs; }; + mastodon = mastodonS3KeyConfig { inherit pkgs; }; + peertube = peertubeS3KeyConfig { inherit pkgs; }; + }; +} diff --git a/launch/main.tf b/launch/main.tf new file mode 100644 index 00000000..ecc64951 --- /dev/null +++ b/launch/main.tf @@ -0,0 +1,179 @@ +variable "domain" { + type = string + default = "fediversity.net" +} + +variable "mastodon" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "pixelfed" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "peertube" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "initialUser" { + type = object({ + displayName = string + username = string + email = string + # TODO: mark (nested) credentials as sensitive + # https://discuss.hashicorp.com/t/is-it-possible-to-mark-an-attribute-of-an-object-as-sensitive/24649/2 + password = string + }) + default = { + displayName = "Testy McTestface" + username = "test" + email = "test@test.com" + password = "testtest" + } +} + +variable "ssh_private_key_file" { + type = string + description = "Path to private key used to connect to the target_host" + default = "" +} + +variable "deploy_environment" { + type = map(string) + description = "Extra environment variables to be set during deployment." + default = {} +} + +locals { + system = "x86_64-linux" + pins = jsondecode(file("${path.module}/.npins.json")) + peripheral_configs = { + garage = "test01" + } + application_configs = { + mastodon = { + cfg = var.mastodon + hostname = "test06" + } + pixelfed = { + cfg = var.pixelfed + hostname = "test04" + } + peertube = { + cfg = var.peertube + hostname = "test05" + } + } + peripherals = { for name, inst in local.peripheral_configs : name => { + hostname = inst + cfg = { + enable = anytrue([for _, app in local.application_configs: app.cfg.enable]) + } + } + } +} + +# FIXME settle for pwd when in /nix/store? +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo '{\"hash\":\"$(nix-hash ..)\"}'"] +} + +# merged instantiate/deploy to prevent 24+s instantiates when nothing changed. +# terraform-nixos separates these to only deploy if instantiate changed. +# FIXME find a better solution for this. current considerations were: +# - generic resources cannot have outputs, while we want info from the instantiation (unless built on host?). +# - `data` always runs, which is slow for deploy/instantiation. +resource "terraform_data" "nixos" { + for_each = {for name, inst in merge( + local.peripherals, + local.application_configs, + ) : name => inst if inst.cfg.enable} + + triggers_replace = [ + data.external.hash.result, + var.deploy_environment, + var.domain, + var.initialUser, + local.system, + each.key, + each.value, + ] + + provisioner "local-exec" { + working_dir = path.root + environment = merge(var.deploy_environment, { + NIX_PATH = join(":", [for name, path in local.pins : "${name}=${path}"]), + }) + # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs? + # TODO: build on target? + command = <<-EOF + set -euo pipefail + + # INSTANTIATE + command=( + nix-instantiate + --expr + 'let + os = import { + system = "${local.system}"; + configuration = { + terraform = builtins.fromJSON "${replace(jsonencode({ + domain = var.domain + hostname = each.value.hostname + initialUser = var.initialUser + }), "\"", "\\\"")}"; + imports = [ + ${path.root}/options.nix + ${path.root}/shared.nix + ${path.root}/${each.key}.nix + # FIXME: get VM details from TF + ${path.root}/../infra/test-machines/${each.value.hostname} + ]; + }; + }; + in { + substituters = builtins.concatStringsSep " " os.config.nix.settings.substituters; + trusted_public_keys = builtins.concatStringsSep " " os.config.nix.settings.trusted-public-keys; + drv_path = os.config.system.build.toplevel.drvPath; + out_path = os.config.system.build.toplevel; + }' + ) + "$${command[@]}" -A out_path + json="$("$${command[@]}" --eval --strict --json)" + + # DEPLOY + declare substituters trusted_public_keys drv_path + eval "export $(echo $json | jaq -r 'to_entries | map("\(.key)=\(.value)") | @sh')" + host="root@${each.value.hostname}.abundos.eu" # FIXME: #24 + buildArgs=( + --option extra-binary-caches https://cache.nixos.org/ + --option substituters $substituters + --option trusted-public-keys $trusted_public_keys + ) + sshOpts=( + -o StrictHostKeyChecking=no + -o BatchMode=yes + -o "IdentityFile='${var.ssh_private_key_file}'" + ) + outPath=$(nix-store --realize "$drv_path" "$${buildArgs[@]}") + NIX_SSHOPTS="$${sshOpts[*]}" nix-copy-closure --to "$host" "$outPath" --gzip --use-substitutes + ssh "$${sshOpts[@]}" "$host" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; $outPath/bin/switch-to-configuration switch" + EOF + } +} diff --git a/launch/mastodon.nix b/launch/mastodon.nix new file mode 100644 index 00000000..26682f50 --- /dev/null +++ b/launch/mastodon.nix @@ -0,0 +1,17 @@ +{ pkgs, ... }: +let + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; +in +{ + fediversity = { + mastodon = mastodonS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + temp.cores = 1; # FIXME: should come from NixOps4 eventually + }; +} diff --git a/launch/options.nix b/launch/options.nix new file mode 100644 index 00000000..c090b372 --- /dev/null +++ b/launch/options.nix @@ -0,0 +1,53 @@ +{ + lib, + ... +}: +let + inherit (lib) types mkOption; + inherit (types) str enum submodule; +in +{ + options.terraform = { + domain = mkOption { + type = enum [ + "fediversity.net" + ]; + description = '' + Apex domain under which the services will be deployed. + ''; + default = "fediversity.net"; + }; + hostname = mkOption { + type = str; + description = '' + Internal name of the host, e.g. test01 + ''; + }; + initialUser = mkOption { + description = '' + Some services require an initial user to access them. + This option sets the credentials for such an initial user. + ''; + type = submodule { + options = { + displayName = mkOption { + type = str; + description = "Display name of the user"; + }; + username = mkOption { + type = str; + description = "Username for login"; + }; + email = mkOption { + type = str; + description = "User's email address"; + }; + password = mkOption { + type = str; + description = "Password for login"; + }; + }; + }; + }; + }; +} diff --git a/launch/pass-ssh-key.sh b/launch/pass-ssh-key.sh new file mode 100755 index 00000000..80f17dec --- /dev/null +++ b/launch/pass-ssh-key.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +export host="$host" + +mkdir -p etc/ssh + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +for keyname in ssh_host_ed25519_key ssh_host_ed25519_key.pub; do + if [[ $keyname == *.pub ]]; then + umask 0133 + else + umask 0177 + fi + cp "$SCRIPT_DIR/../infra/test-machines/${host}/${keyname}" ./etc/ssh/${keyname} +done diff --git a/launch/peertube.nix b/launch/peertube.nix new file mode 100644 index 00000000..4124568a --- /dev/null +++ b/launch/peertube.nix @@ -0,0 +1,20 @@ +{ pkgs, ... }: +let + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; +in +{ + fediversity = { + peertube = peertubeS3KeyConfig { inherit pkgs; } // { + enable = true; + ## NOTE: Only ever used for testing anyway. + ## + ## FIXME: Generate and store in NixOps4's state. + secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24"; + }; + }; +} diff --git a/launch/pixelfed.nix b/launch/pixelfed.nix new file mode 100644 index 00000000..75790409 --- /dev/null +++ b/launch/pixelfed.nix @@ -0,0 +1,16 @@ +{ pkgs, ... }: +let + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +{ + fediversity = { + pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + }; +} diff --git a/launch/resource.nix b/launch/resource.nix new file mode 100644 index 00000000..7ae3f99f --- /dev/null +++ b/launch/resource.nix @@ -0,0 +1,44 @@ +{ + lib, + config, + ... +}: + +let + inherit (lib) attrValues elem mkDefault; + inherit (lib.attrsets) concatMapAttrs optionalAttrs; + inherit (lib.strings) removeSuffix; + + secretsPrefix = ../secrets; + secrets = import (secretsPrefix + "/secrets.nix"); + keys = import ../keys; + +in +{ + fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name}; + + ## The configuration of the machine. We strive to keep in this file only the + ## options that really need to be injected from the resource. Everything else + ## should go into the `./nixos` subdirectory. + imports = [ + ../infra/common/options.nix + ../infra/common/nixos + ]; + + ## Read all the secrets, filter the ones that are supposed to be readable + ## with this host's public key, and add them correctly to the configuration + ## as `age.secrets..file`. + age.secrets = concatMapAttrs ( + name: secret: + optionalAttrs (elem config.fediversityVm.hostPublicKey secret.publicKeys) { + ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}"; + } + ) secrets; + + ## FIXME: Remove direct root authentication once the NixOps4 NixOS provider + ## supports users with password-less sudo. + users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors ++ [ + # allow our panel vm access to the test machines + keys.panel + ]; +} diff --git a/launch/shared.nix b/launch/shared.nix new file mode 100644 index 00000000..46d6ccc5 --- /dev/null +++ b/launch/shared.nix @@ -0,0 +1,26 @@ +{ + pkgs, + config, + ... +}: +let + inherit (config.terraform) hostname domain initialUser; +in +{ + imports = [ + + + ../services/fediversity + ./resource.nix + ]; + fediversityVm.name = hostname; + fediversity = { + inherit domain; + temp.initialUser = { + inherit (initialUser) username email displayName; + # FIXME: disgusting, but nvm, this is going to be replaced by + # proper central authentication at some point + passwordFile = pkgs.writeText "password" initialUser.password; + }; + }; +} diff --git a/launch/shell.nix b/launch/shell.nix new file mode 100644 index 00000000..a6bdf202 --- /dev/null +++ b/launch/shell.nix @@ -0,0 +1 @@ +(import ./. { }).shell diff --git a/launch/tf-env.nix b/launch/tf-env.nix new file mode 100644 index 00000000..9924a917 --- /dev/null +++ b/launch/tf-env.nix @@ -0,0 +1,26 @@ +{ + lib, + pkgs, + sources ? import ../npins, + ... +}: +pkgs.stdenv.mkDerivation { + name = "tf-repo"; + src = ../.; + buildInputs = [ + (import ./tf.nix { inherit lib pkgs; }) + ]; + buildPhase = '' + runHook preBuild + pushd launch/ + echo '${lib.strings.toJSON sources}' > .npins.json + tofu init -input=false + popd + runHook postBuild + ''; + installPhase = '' + runHook preInstall + cp -r . $out + runHook postInstall + ''; +} diff --git a/launch/tf.nix b/launch/tf.nix new file mode 100644 index 00000000..14ad72a7 --- /dev/null +++ b/launch/tf.nix @@ -0,0 +1,25 @@ +# FIXME: use overlays so this gets imported just once? +{ + lib, + pkgs, + ... +}: +let + tofuProvider = + provider: + provider.override (oldArgs: { + provider-source-address = + lib.replaceStrings [ "https://registry.terraform.io/providers" ] [ "registry.opentofu.org" ] + oldArgs.homepage; + }); + tf = pkgs.opentofu; + tfPlugins = ( + p: [ + p.null + p.external + ] + ); +in +# tf.withPlugins tfPlugins +# https://github.com/NixOS/nixpkgs/pull/358522 +tf.withPlugins (p: pkgs.lib.lists.map tofuProvider (tfPlugins p)) diff --git a/npins/sources.json b/npins/sources.json index 45efd6b4..bedab8ae 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -1,5 +1,44 @@ { "pins": { + "agenix": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "ryantm", + "repo": "agenix" + }, + "branch": "main", + "revision": "e600439ec4c273cf11e06fe4d9d906fb98fa097c", + "url": "https://github.com/ryantm/agenix/archive/e600439ec4c273cf11e06fe4d9d906fb98fa097c.tar.gz", + "hash": "006ngydiykjgqs85cl19h9klq8kaqm5zs0ng51dnwy7nzgqxzsdr" + }, + "disko": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "nix-community", + "repo": "disko" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "version": "v1.11.0", + "revision": "cdf8deded8813edfa6e65544f69fdd3a59fa2bb4", + "url": "https://api.github.com/repos/nix-community/disko/tarball/v1.11.0", + "hash": "13brimg7z7k9y36n4jc1pssqyw94nd8qvgfjv53z66lv4xkhin92" + }, + "flake-inputs": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "fricklerhandwerk", + "repo": "flake-inputs" + }, + "branch": "main", + "revision": "559574c9cbb8af262f3944b67d60fbf0f6ad03c3", + "url": "https://github.com/fricklerhandwerk/flake-inputs/archive/559574c9cbb8af262f3944b67d60fbf0f6ad03c3.tar.gz", + "hash": "0gbhmp6x2vdzvfnsvqzal3g8f8hx2ia6r73aibc78kazf78m67x6" + }, "htmx": { "type": "GitRelease", "repository": { @@ -30,8 +69,8 @@ "nixpkgs": { "type": "Channel", "name": "nixpkgs-unstable", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre711046.8edf06bea5bc/nixexprs.tar.xz", - "hash": "1mwsn0rvfm603svrq3pca4c51zlix5gkyr4gl6pxhhq3q6xs5s8y" + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre777917.b7ba7f9f45c5/nixexprs.tar.xz", + "hash": "0jb6b7sv66bn06pchj2l88z0i5dlz0c2vb3z6pjjlq2p8q11zigg" } }, "version": 3 diff --git a/panel/default.nix b/panel/default.nix index 767802be..06d1a487 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -20,11 +20,18 @@ in pkgs.npins manage ]; - env = import ./env.nix { inherit lib pkgs; } // { - NPINS_DIRECTORY = toString ../npins; - CREDENTIALS_DIRECTORY = toString ./.credentials; - DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; - }; + env = + let + inherit (builtins) toString; + in + import ./env.nix { inherit lib pkgs; } + // { + NPINS_DIRECTORY = toString ../npins; + CREDENTIALS_DIRECTORY = toString ./.credentials; + DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; + # locally: use a fixed relative reference, so we can use our newest files without copying to the store + REPO_DIR = toString ../.; + }; shellHook = '' ln -sf ${sources.htmx}/dist/htmx.js src/panel/static/htmx.min.js # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd. diff --git a/panel/env.nix b/panel/env.nix index 07ce4193..90d15d7a 100644 --- a/panel/env.nix +++ b/panel/env.nix @@ -3,16 +3,15 @@ pkgs, ... }: -let - inherit (builtins) toString; -in { - REPO_DIR = toString ../.; - # explicitly use nix, as e.g. lix does not have configurable-impure-env BIN_PATH = lib.makeBinPath [ - # explicitly use nix, as e.g. lix does not have configurable-impure-env - pkgs.nix - # nixops error maybe due to our flake git hook: executing 'git': No such file or directory + pkgs.lix + pkgs.bash + pkgs.coreutils + pkgs.openssh pkgs.git + pkgs.jaq # tf + (import ../launch/tf.nix { inherit lib pkgs; }) ]; + SSH_PRIVATE_KEY_FILE = ""; } diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index ecf06e0f..6061e559 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -29,6 +29,8 @@ let ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) (builtins.toFile "extra-settings.py" cfg.extra-settings) ]; + REPO_DIR = import ../../launch/tf-env.nix { inherit lib pkgs; }; + SSH_PRIVATE_KEY_FILE = config.age.secrets.panel-ssh-key.path; }; python-environment = pkgs.python3.withPackages ( @@ -157,16 +159,14 @@ in }; }; - users.users.${name} = { - isNormalUser = true; - }; + users.users.${name}.isNormalUser = true; - users.groups.${name} = { }; systemd.services.${name} = { description = "${name} ASGI server"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = [ + pkgs.openssh python-environment manage-service ]; diff --git a/panel/nix/package.nix b/panel/nix/package.nix index 9337887c..0acd9796 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -1,5 +1,6 @@ { lib, + pkgs, sqlite, python3, sources ? import ../../npins, @@ -11,7 +12,7 @@ let root = ../src; fileset = intersection (gitTracked ../../.) ../src; }; - pyproject = with lib; fromTOML pyproject-toml; + pyproject = fromTOML pyproject-toml; # TODO: define this globally name = "panel"; # TODO: we may want this in a file so it's easier to read statically @@ -58,7 +59,9 @@ python3.pkgs.buildPythonPackage { mkdir -p $out/bin cp -v ${src}/manage.py $out/bin/manage.py chmod +x $out/bin/manage.py - wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH" + wrapProgram $out/bin/manage.py \ + --set REPO_DIR "${import ../../launch/tf-env.nix { inherit lib pkgs; }}" \ + --prefix PYTHONPATH : "$PYTHONPATH" cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/ ''; } diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index b270612c..91aa5824 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -10,14 +10,19 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/4.2/ref/settings/ """ +import re import sys +import subprocess import os +import json import importlib.util import dj_database_url from os import environ as env from pathlib import Path +STORE_PATTERN = re.compile("^/nix/store/[^/]+$") + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -171,6 +176,53 @@ COMPRESS_PRECOMPILERS = [ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", + }, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", + }, + }, + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", + }, + "standard": { + "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s" + }, + }, + "handlers": { + "console": { + "level": "INFO", + # "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "standard", + }, + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", + }, + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "DEBUG" if DEBUG else "INFO", + }, + }, +} + # Customization via user settings # This must be at the end, as it must be able to override the above # TODO(@fricklerhandwerk): @@ -197,3 +249,12 @@ bin_path=env['BIN_PATH'] # path of the root flake to trigger nixops from, see #94. # to deploy this should be specified, for dev just use a relative path. repo_dir = env["REPO_DIR"] + +output = subprocess.run(["ssh-agent"], capture_output=True, text=True, env={"PATH": bin_path}).stdout +ssh_auth_sock = re.search("(?<==)([^;]*)", output)[1] +ENV_VARS = { + "ssh_private_key_file": env["SSH_PRIVATE_KEY_FILE"], + "deploy_environment": { + "SSH_AUTH_SOCK": ssh_auth_sock, + }, +} diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 5c423fc6..e39d0f70 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -1,6 +1,8 @@ from enum import Enum import json +from os.path import expanduser import subprocess +import logging import os from django.urls import reverse_lazy @@ -13,6 +15,8 @@ from django.shortcuts import render from panel import models, settings from panel.configuration import forms +logger = logging.getLogger(__name__) + class Index(TemplateView): template_name = 'index.html' @@ -102,7 +106,7 @@ class DeploymentStatus(ConfigurationForm): # Check for deploy button if "deploy" in self.request.POST.keys(): deployment_result, deployment_params = self.deployment(obj) - deployment_succeeded = deployment_result.returncode == 0 + deployment_succeeded = deployment_result == 0 return render(self.request, "partials/deployment_result.html", { "deployment_succeeded": deployment_succeeded, @@ -134,25 +138,26 @@ class DeploymentStatus(ConfigurationForm): }, } # serialize back and forth now we still need to manually inject the dummy user - deployment_params = json.dumps(dummy_user | json.loads(submission)) + deployment_params = dummy_user | json.loads(submission) env = { "PATH": settings.bin_path, + # "TF_LOG": "info", + } | { # pass in form info to our deployment - "DEPLOYMENT": deployment_params, + # FIXME: ensure sensitive info is protected + f"TF_VAR_{k}": v if isinstance(v, str) else json.dumps(v) for k, v in (settings.ENV_VARS | deployment_params).items() } + logger.info("env: %s", env) + cwd = f"{settings.repo_dir}/launch" cmd = [ - "nix", - "develop", - "--extra-experimental-features", - "configurable-impure-env", - "--command", - "nixops4", + "tofu", + # f"-chdir={cwd}", "apply", - "test", + f"-state={cwd}/terraform.tfstate", # FIXME: separate users' state + "--auto-approve", + "-lock=false", + "-parallelism=1" # limit OOM risk ] - deployment_result = subprocess.run( - cmd, - cwd=settings.repo_dir, - env=env, - ) - return deployment_result, json.loads(deployment_params) + deployment_result = subprocess.run(cmd, cwd=cwd, env=env) + logger.debug("deployment_result: %s", deployment_result) + return deployment_result, deployment_params