From dd5a6335b15b58adcc1e91577d2ced1504ed7fab Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Thu, 1 May 2025 15:03:06 +0200 Subject: [PATCH] proxmox pass in description fix syntax configure proxmox provider typo add doc comment in existing modules add comment allow insecure proxmox connection for use in dev wip proxmox progress use service configurations moved to machine-independent location wire settings directly without option block terraform adjust cwd try tf on null input update .envrc.sample with sample proxmox credentials --- .forgejo/workflows/ci.yaml | 8 +- .gitignore | 6 + README.md | 30 +- default.nix | 4 + deployment/README.md | 6 - infra/.envrc.sample | 1 + infra/.gitignore | 1 + infra/README.md | 101 ++----- infra/common/options.nix | 4 +- infra/common/resource.nix | 6 +- infra/common/shared.nix | 1 + infra/default.nix | 30 ++ infra/dev/main.tf | 44 +++ infra/dev/options.nix | 29 ++ infra/dev/variables.tf | 1 + infra/machines.md | 15 - infra/machines.md.sh | 43 --- infra/machines/vm02116/default.nix | 38 --- infra/machines/vm02187/default.nix | 36 --- infra/old-deployment-readme.md | 223 --------------- infra/operator/garage.nix | 34 +++ infra/operator/main.tf | 90 ++++++ infra/operator/mastodon.nix | 17 ++ infra/operator/options.nix | 55 ++++ infra/operator/peertube.nix | 20 ++ infra/operator/pixelfed.nix | 16 ++ infra/operator/variables.tf | 51 ++++ infra/pass-ssh-key.sh | 15 + infra/proxmox-remove.sh | 258 ------------------ infra/setup.nix | 20 ++ infra/shell.nix | 1 + infra/sync-nix/main.tf | 224 +++++++++++++++ infra/sync-nix/variables.tf | 43 +++ infra/test-machines/test01/default.nix | 19 -- infra/test-machines/test02/default.nix | 19 -- infra/test-machines/test03/default.nix | 19 -- infra/test-machines/test04/default.nix | 19 -- infra/test-machines/test05/default.nix | 19 -- infra/test-machines/test06/default.nix | 19 -- infra/test-machines/test11/default.nix | 19 -- infra/test-machines/test12/default.nix | 19 -- infra/test-machines/test13/default.nix | 19 -- infra/test-machines/test14/default.nix | 19 -- infra/tests.nix | 37 +++ infra/tf-env.nix | 32 +++ infra/tf.nix | 34 +++ keys/README.md | 2 +- machines/README.md | 4 + .../dev}/fedi200/default.nix | 0 machines/dev/fedi200/dns.nix | 2 + .../dev}/fedi201/default.nix | 6 - .../dev}/fedi201/fedipanel.nix | 20 +- machines/dev/vm02116/default.nix | 31 +++ .../dev}/vm02116/forgejo.nix | 0 machines/dev/vm02187/default.nix | 29 ++ .../dev}/vm02187/wiki.nix | 0 .../operator}/configuration.json | 0 .../operator}/test01/ssh_host_ed25519_key | 0 .../operator}/test01/ssh_host_ed25519_key.pub | 0 .../operator}/test02/ssh_host_ed25519_key | 0 .../operator}/test02/ssh_host_ed25519_key.pub | 0 .../operator}/test03/ssh_host_ed25519_key | 0 .../operator}/test03/ssh_host_ed25519_key.pub | 0 .../operator}/test04/ssh_host_ed25519_key | 0 .../operator}/test04/ssh_host_ed25519_key.pub | 0 .../operator}/test05/ssh_host_ed25519_key | 0 .../operator}/test05/ssh_host_ed25519_key.pub | 0 .../operator}/test06/ssh_host_ed25519_key | 0 .../operator}/test06/ssh_host_ed25519_key.pub | 0 .../operator}/test11/ssh_host_ed25519_key | 0 .../operator}/test11/ssh_host_ed25519_key.pub | 0 .../operator}/test12/ssh_host_ed25519_key | 0 .../operator}/test12/ssh_host_ed25519_key.pub | 0 .../operator}/test13/ssh_host_ed25519_key | 0 .../operator}/test13/ssh_host_ed25519_key.pub | 0 .../operator}/test14/ssh_host_ed25519_key | 0 .../operator}/test14/ssh_host_ed25519_key.pub | 0 npins/sources.json | 32 ++- panel/default.nix | 17 +- panel/env.nix | 16 +- panel/nix/configuration.nix | 8 +- panel/nix/package.nix | 11 +- .../django-pydantic-field/default.nix | 1 + panel/src/panel/settings.py | 8 +- panel/src/panel/views.py | 35 ++- {infra => proxmox}/makeInstallerIso.nix | 1 - secrets/.envrc | 10 + secrets/README.md | 2 +- secrets/default.nix | 24 ++ secrets/shell.nix | 1 + services/.envrc | 10 + services/fediversity/default.nix | 2 +- services/fediversity/peertube/options.nix | 2 +- 93 files changed, 1064 insertions(+), 974 deletions(-) delete mode 100644 deployment/README.md create mode 100644 infra/.envrc.sample create mode 100644 infra/.gitignore create mode 100644 infra/default.nix create mode 100644 infra/dev/main.tf create mode 100644 infra/dev/options.nix create mode 100644 infra/dev/variables.tf delete mode 100644 infra/machines.md delete mode 100644 infra/machines.md.sh delete mode 100644 infra/machines/vm02116/default.nix delete mode 100644 infra/machines/vm02187/default.nix delete mode 100644 infra/old-deployment-readme.md create mode 100644 infra/operator/garage.nix create mode 100644 infra/operator/main.tf create mode 100644 infra/operator/mastodon.nix create mode 100644 infra/operator/options.nix create mode 100644 infra/operator/peertube.nix create mode 100644 infra/operator/pixelfed.nix create mode 100644 infra/operator/variables.tf create mode 100755 infra/pass-ssh-key.sh delete mode 100755 infra/proxmox-remove.sh create mode 100644 infra/setup.nix create mode 100644 infra/shell.nix create mode 100644 infra/sync-nix/main.tf create mode 100644 infra/sync-nix/variables.tf delete mode 100644 infra/test-machines/test01/default.nix delete mode 100644 infra/test-machines/test02/default.nix delete mode 100644 infra/test-machines/test03/default.nix delete mode 100644 infra/test-machines/test04/default.nix delete mode 100644 infra/test-machines/test05/default.nix delete mode 100644 infra/test-machines/test06/default.nix delete mode 100644 infra/test-machines/test11/default.nix delete mode 100644 infra/test-machines/test12/default.nix delete mode 100644 infra/test-machines/test13/default.nix delete mode 100644 infra/test-machines/test14/default.nix create mode 100644 infra/tests.nix create mode 100644 infra/tf-env.nix create mode 100644 infra/tf.nix create mode 100644 machines/README.md rename {infra/machines => machines/dev}/fedi200/default.nix (100%) create mode 100644 machines/dev/fedi200/dns.nix rename {infra/machines => machines/dev}/fedi201/default.nix (82%) rename {infra/machines => machines/dev}/fedi201/fedipanel.nix (58%) create mode 100644 machines/dev/vm02116/default.nix rename {infra/machines => machines/dev}/vm02116/forgejo.nix (100%) create mode 100644 machines/dev/vm02187/default.nix rename {infra/machines => machines/dev}/vm02187/wiki.nix (100%) rename {infra/test-machines => machines/operator}/configuration.json (100%) rename {infra/test-machines => machines/operator}/test01/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test01/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test02/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test02/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test03/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test03/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test04/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test04/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test05/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test05/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test06/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test06/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test11/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test11/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test12/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test12/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test13/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test13/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test14/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test14/ssh_host_ed25519_key.pub (100%) rename {infra => proxmox}/makeInstallerIso.nix (99%) create mode 100644 secrets/.envrc create mode 100644 secrets/default.nix create mode 100644 secrets/shell.nix create mode 100644 services/.envrc diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 98b83852..57e8ef6e 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - run: nix-build -A tests - check-peertube: + check-services: runs-on: native steps: - uses: actions/checkout@v4 @@ -32,3 +32,9 @@ jobs: steps: - uses: actions/checkout@v4 - run: nix build .#checks.x86_64-linux.deployment-basic -L + + check-infra: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: cd infra && nix-build -A tests diff --git a/.gitignore b/.gitignore index 7aa68143..cafa559c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ +.npins.json +.terraform/ +.terraform.lock.hcl +.terraform.tfstate.lock.info +terraform.tfstate* +.auto.tfvars.json .DS_Store .idea *.log diff --git a/README.md b/README.md index 877d2e2e..09ecadef 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # The Fediversity project This repository contains all the code and code-related files having to do with -[the Fediversity project](https://fediversity.eu/), with the notable exception -of [NixOps4 that is hosted on GitHub](https://github.com/nixops4/nixops4). +[the Fediversity project](https://fediversity.eu/). ## Goals @@ -81,27 +80,15 @@ Not everyone has the expertise and time to run their own server. The software includes technical configuration that links software components. Most user-facing configuration remains untouched by the deployment process. - > Example: NixOps4 is used to deploy [Pixelfed](https://pixelfed.org). + > Example: OpenTofu is used to deploy [Pixelfed](https://pixelfed.org). - Migrate Move service configurations and user data to a different hosting provider. -- [NixOps4](https://github.com/nixops4/nixops4) +- [OpenTofu](https://opentofu.org/) - A tool for deploying and managing resources through the Nix language. - NixOps4 development is supported by the Fediversity project - -- Resource - - A [resource for NixOps4](https://nixops.dev/manual/development/concept/resource.html) is any external entity that can be declared with NixOps4 expressions and manipulated with NixOps4, such as a virtual machine, an active NixOS configuration, a DNS entry, or customer database. - -- Resource provider - - A resource provider for NixOps4 is an executable that communicates between a resource and NixOps4 using a standardised protocol, allowing [CRUD operations](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) on the resources to be performed by NixOps4. - Refer to the [NixOps4 manual](https://nixops.dev/manual/development/resource-provider/index.html) for details. - - > Example: We need a resource provider for obtaining deployment secrets from a database. + An infrastructure-as-code tool, and open-source (MPL 2.0) fork of Terraform. ## Development @@ -118,9 +105,6 @@ Contact the project team if you have questions or suggestions, or if you're inte Most of the directories in this repository have their own README going into more details as to what they are for. As an overview: -- [`deployment/`](./deployment) contains work to generate a full Fediversity - deployment from a minimal configuration. - - [`infra/`](./infra) contains the configurations for the various VMs that are in production for the project, for instance the Git instances or the Wiki, as well as means to provision and set up new ones. @@ -128,14 +112,8 @@ details as to what they are for. As an overview: - [`keys/`](./keys) contains the public keys of the contributors to this project as well as the systems that we administrate. -- [`matrix/`](./matrix) contains everything having to do with setting up a - fully-featured Matrix server. - - [`secrets/`](./secrets) contains the secrets that need to get injected into machine configurations. - [`services/`](./services) contains our effort to make Fediverse applications work seemlessly together in our specific setting. - -- [`website/`](./website) contains the framework and the content of [the - Fediversity website](https://fediversity.eu/) diff --git a/default.nix b/default.nix index 4c71ec49..c89502b3 100644 --- a/default.nix +++ b/default.nix @@ -24,6 +24,7 @@ let ## Add a directory here if pre-commit hooks shouldn't apply to it. optout = [ "npins" + ".terraform" ]; excludes = map (dir: "^${dir}/") optout; addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); @@ -41,6 +42,9 @@ in shell = pkgs.mkShellNoCC { inherit (pre-commit-check) shellHook; buildInputs = pre-commit-check.enabledPackages; + packages = [ + pkgs.nixfmt-rfc-style + ]; }; tests = { diff --git a/deployment/README.md b/deployment/README.md deleted file mode 100644 index 4d7d039b..00000000 --- a/deployment/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Deployment - -This repository contains work to generate a full Fediversity deployment from a -minimal configuration. This is different from [`../services/`](../services) that -focuses on one machine, providing a polished and unified interface to different -Fediverse services. diff --git a/infra/.envrc.sample b/infra/.envrc.sample new file mode 100644 index 00000000..2e5385f8 --- /dev/null +++ b/infra/.envrc.sample @@ -0,0 +1 @@ +export PROXMOX_VE_API_TOKEN="myuser@ProcoliX!TOKEN_NAME=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 00000000..ea289fc8 --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1 @@ +**/.envrc diff --git a/infra/README.md b/infra/README.md index 133f6a32..3f1e4060 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,98 +1,37 @@ -# Infra +# service deployment -This directory contains the definition of [the VMs](machines.md) that host our -infrastructure. +deploys [NixOS](https://nixos.org/) templates using [OpenTofu](https://opentofu.org/). -## Provisioning VMs with an initial configuration +## requirements -NOTE[Niols]: This is very manual and clunky. Two things will happen. In the near -future, I will improve the provisioning script to make this a bit less clunky. -In the far future, NixOps4 will be able to communicate with Proxmox directly and -everything will become much cleaner. +- [nix](https://nix.dev/) -1. Choose names for your VMs. It is recommended to choose `fediXXX`, with `XXX` - above 100. For instance, `fedi117`. +## usage -2. Add a basic configuration for the machine. These typically go in - `infra/machines//default.nix`. You can look at other `fediXXX` VMs to - find inspiration. You probably do not need a `nixos.module` option at this - point. +### development -2. Add a file for each of those VM's public keys, eg. - ``` - touch keys/systems/fedi117.pub - ``` - Those files need to exist during provisioning, but their content matters only - when updating the machines' configuration. - - FIXME: Remove this step by making the provisioning script not fail with the - public key does not exist yet. - -3. Run the provisioning script: - ``` - sh infra/proxmox-provision.sh fedi117 - ``` - The script can take several ids at the same time. It requires some - authentication options and provides several more. See `--help`. - -4. (Optional) Add a DNS entry for the machine; for instance `fedi117.abundos.eu - A 95.215.187.117`. - -5. Grab the public host keys for the machines in question, and add it to the - repository. For instance: - ``` - ssh fedi117.abundos.eu 'sudo cat /etc/ssh/ssh_host_ed25519_key.pub' > keys/systems/fedi117.pub - ``` - - FIXME: Make the provisioning script do that for us. - -7. Regenerate the list of machines: - ``` - sh infra/machines.md.sh - ``` - Commit it with the machine's configuration, public key, etc. - -8. At this point, the machine contains a very basic configuration that contains - just enough for it to boot and be reachable. Go on to the next section to - update the machine and put an actual configuration. - - FIXME: Figure out why the full configuration isn't on the machine at this - point and fix it. - -## Updating existing VM configurations - -Their configuration can be updated via NixOps4. Run +before using other commands, if not using direnv: ```sh -nixops4 deployments list +nix-shell ``` -to see the available deployments. -This should be done from the root of the repository, -otherwise NixOps4 will fail with something like: - -``` - nixops4 error: evaluation: error: - … while calling the 'getFlake' builtin - - error: path '/nix/store/05nn7krhvi8wkcyl6bsysznlv60g5rrf-source/flake.nix' does not exist, evaluation: error: - … while calling the 'getFlake' builtin - - error: path '/nix/store/05nn7krhvi8wkcyl6bsysznlv60g5rrf-source/flake.nix' does not exist -``` - -Then, given a deployment (eg. `fedi200`), run +then to initialize, or after updating pins or TF providers: ```sh -nixops4 apply +setup ``` -Alternatively, to run the `default` deployment, which contains all the VMs, run +then, one can use the `tofu` CLI in the sub-folders. -```sh -nixops4 apply -``` +## credentials -## Removing an existing VM +Credentials may be placed in (sub-folders') `.envrc`, see: -See `infra/proxmox-remove.sh --help`. +- `.envrc.sample` +- TF [proxmox provider](https://registry.terraform.io/providers/bpg/proxmox/latest/docs#environment-variables-summary) + +## implementing + +proper documentation TODO. +until then, a reference implementation may be found in [`panel/`](https://git.fediversity.eu/Fediversity/Fediversity/src/branch/main/panel). diff --git a/infra/common/options.nix b/infra/common/options.nix index 7ccb1611..382c2a1e 100644 --- a/infra/common/options.nix +++ b/infra/common/options.nix @@ -23,7 +23,7 @@ in description = '' The IP address of the machine, version 4. It will be injected as a value in `networking.interfaces.eth0`, but it will also be used to - communicate with the machine via NixOps4. + communicate with the machine. ''; }; @@ -48,7 +48,7 @@ in description = '' The IP address of the machine, version 6. It will be injected as a value in `networking.interfaces.eth0`, but it will also be used to - communicate with the machine via NixOps4. + communicate with the machine. ''; }; diff --git a/infra/common/resource.nix b/infra/common/resource.nix index 390672d9..b71d85c4 100644 --- a/infra/common/resource.nix +++ b/infra/common/resource.nix @@ -21,7 +21,6 @@ in ]; fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name}; - ## Read all the secrets, filter the ones that are supposed to be readable ## with this host's public key, and add them correctly to the configuration ## as `age.secrets..file`. @@ -34,8 +33,5 @@ in ## FIXME: Remove direct root authentication once the NixOps4 NixOS provider ## supports users with password-less sudo. - users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors ++ [ - # allow our panel vm access to the test machines - keys.panel - ]; + users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors; } diff --git a/infra/common/shared.nix b/infra/common/shared.nix index 735ae034..208196ff 100644 --- a/infra/common/shared.nix +++ b/infra/common/shared.nix @@ -10,6 +10,7 @@ in { imports = [ "${sources.agenix}/modules/age.nix" + "${sources.disko}/module.nix" ../../services/fediversity ./resource.nix ]; diff --git a/infra/default.nix b/infra/default.nix new file mode 100644 index 00000000..a4cbfb1a --- /dev/null +++ b/infra/default.nix @@ -0,0 +1,30 @@ +{ + system ? builtins.currentSystem, + sources ? import ../npins, + pkgs ? import sources.nixpkgs { inherit system; }, +}: +let + inherit (pkgs) lib; + setup = import ./setup.nix { inherit lib pkgs sources; }; +in +{ + # shell for testing TF directly + shell = pkgs.mkShellNoCC { + packages = [ + (import ./tf.nix { inherit lib pkgs; }) + pkgs.direnv + pkgs.jaq + setup + ]; + }; + + tests = pkgs.callPackage ./tests.nix { }; + + # re-export inputs so they can be overridden granularly + # (they can't be accessed from the outside any other way) + inherit + sources + system + pkgs + ; +} diff --git a/infra/dev/main.tf b/infra/dev/main.tf new file mode 100644 index 00000000..5dde95ba --- /dev/null +++ b/infra/dev/main.tf @@ -0,0 +1,44 @@ +locals { + vm_domain = "abundos.eu" +} + +module "nixos" { + source = "../sync-nix" + + vm_domain = local.vm_domain + hostname = each.value.hostname + config_nix = each.value.config_nix + config_tf = each.value.config_tf + + for_each = { for name, inst in { + # wiki = "vm02187" # does not resolve + # forgejo = "vm02116" # does not resolve + # TODO: move these to a separate `host` dir + dns = "fedi200" + fedipanel = "fedi201" + } : name => { + hostname = inst + config_tf = { + terraform = { + domain = local.vm_domain + hostname = inst + } + } + config_nix = <<-EOF + { + # note interpolations here TF ones + imports = [ + # shared NixOS config + ${path.root}/../common/shared.nix + # FIXME: separate template options by service + ${path.root}/options.nix + # for service `forgejo` import `forgejo.nix` + ${path.root}/../../machines/dev/${inst}/${name}.nix + # FIXME: get VM details from TF + ${path.root}/../../machines/dev/${inst} + ]; + } + EOF + } + } +} diff --git a/infra/dev/options.nix b/infra/dev/options.nix new file mode 100644 index 00000000..46514ea7 --- /dev/null +++ b/infra/dev/options.nix @@ -0,0 +1,29 @@ +# nix options expected to be set from TF here +# TODO: could (part of) this be generated somehow? c.f #275 +{ + lib, + ... +}: +let + inherit (lib) types mkOption; + inherit (types) str enum; +in +{ + options.terraform = { + domain = mkOption { + type = enum [ + "fediversity.net" + ]; + description = '' + Apex domain under which the services will be deployed. + ''; + default = "fediversity.net"; + }; + hostname = mkOption { + type = str; + description = '' + Internal name of the host, e.g. test01 + ''; + }; + }; +} diff --git a/infra/dev/variables.tf b/infra/dev/variables.tf new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/infra/dev/variables.tf @@ -0,0 +1 @@ + diff --git a/infra/machines.md b/infra/machines.md deleted file mode 100644 index e2c49c0c..00000000 --- a/infra/machines.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Machines - -Currently, this repository keeps track of the following VMs: - -Machine | Proxmox | Description ---------|---------|------------- -[`fedi200`](./fedi200) | fediversity | Testing machine for Hans -[`fedi201`](./fedi201) | fediversity | FediPanel -[`vm02116`](./vm02116) | procolix | Forgejo -[`vm02187`](./vm02187) | procolix | Wiki - -This table excludes all machines with names starting with `test`. diff --git a/infra/machines.md.sh b/infra/machines.md.sh deleted file mode 100644 index ea1b0208..00000000 --- a/infra/machines.md.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env sh -set -euC - -cd "$(dirname "$0")" - -{ -cat <<\EOF - - -# Machines - -Currently, this repository keeps track of the following VMs: - -Machine | Proxmox | Description ---------|---------|------------- -EOF - -vmOptions=$( - cd .. - nix eval \ - --impure --raw --expr " - builtins.toJSON (builtins.getFlake (builtins.toString ./.)).vmOptions - " \ - --log-format raw --quiet -) - -## NOTE: `jq`'s `keys` is alphabetically sorted, just what we want here. -for machine in $(echo "$vmOptions" | jq -r 'keys[]'); do - if [ "${machine#test}" = "$machine" ]; then - proxmox=$(echo "$vmOptions" | jq -r ".$machine.proxmox") - description=$(echo "$vmOptions" | jq -r ".$machine.description" | head -n 1) - - # shellcheck disable=SC2016 - printf '[`%s`](./%s) | %s | %s\n' "$machine" "$machine" "$proxmox" "$description" - fi -done - -cat <<\EOF - -This table excludes all machines with names starting with `test`. -EOF -} >| machines.md diff --git a/infra/machines/vm02116/default.nix b/infra/machines/vm02116/default.nix deleted file mode 100644 index be461038..00000000 --- a/infra/machines/vm02116/default.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ - fediversityVm = { - vmId = 2116; - proxmox = "procolix"; - description = "Forgejo"; - - ipv4.address = "185.206.232.34"; - ipv6.address = "2a00:51c0:12:1201::20"; - }; - - nixos.module = - { lib, ... }: - { - imports = [ - ./forgejo.nix - ]; - - ## vm02116 is running on old hardware based on a Xen VM environment, so it - ## needs these extra options. Once the VM gets moved to a newer node, these - ## two options can safely be removed. - boot.initrd.availableKernelModules = [ "xen_blkfront" ]; - services.xe-guest-utilities.enable = true; - - ## NOTE: This VM was created manually, which requires us to override the - ## default disko-based `fileSystems` definition. - fileSystems = lib.mkForce { - "/" = { - device = "/dev/disk/by-uuid/3802a66d-e31a-4650-86f3-b51b11918853"; - fsType = "ext4"; - }; - - "/boot" = { - device = "/dev/disk/by-uuid/2CE2-1173"; - fsType = "vfat"; - }; - }; - }; -} diff --git a/infra/machines/vm02187/default.nix b/infra/machines/vm02187/default.nix deleted file mode 100644 index b389efb2..00000000 --- a/infra/machines/vm02187/default.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ - fediversityVm = { - vmId = 2187; - proxmox = "procolix"; - description = "Wiki"; - - ipv4.address = "185.206.232.187"; - ipv6.address = "2a00:51c0:12:1201::187"; - }; - - nixos.module = - { lib, ... }: - { - imports = [ - ./wiki.nix - ]; - - ## NOTE: This VM was created manually, which requires us to override the - ## default disko-based `fileSystems` definition. - fileSystems = lib.mkForce { - "/" = { - device = "/dev/disk/by-uuid/a46a9c46-e32b-4216-a4aa-8819b2cd0d49"; - fsType = "ext4"; - }; - - "/boot" = { - device = "/dev/disk/by-uuid/6AB5-4FA8"; - fsType = "vfat"; - options = [ - "fmask=0022" - "dmask=0022" - ]; - }; - }; - }; -} diff --git a/infra/old-deployment-readme.md b/infra/old-deployment-readme.md deleted file mode 100644 index f4899022..00000000 --- a/infra/old-deployment-readme.md +++ /dev/null @@ -1,223 +0,0 @@ -# Provisioning VMs via Proxmox - -NOTE: This directory is outdated and most of the interesting code has moved to -`infra/`. There is still some information to extract from here, but treat all -that you read with a grain of salt. - -## Quick links - -Proxmox API doc -: - -Fediversity Proxmox -: - -## Basic terminology - -Node -: physical host - -## Fediversity Proxmox - -- It is only accessible via Procolix\'s VPN: - - Get credentials for the VPN portal and Proxmox from - [Kevin](https://git.fediversity.eu/kevin). - - - Log in to the [VPN - portal](https://vpn.fediversity.eu/vpn-user-portal/home). - - - Create a **New Configuration**: - - Select **WireGuard (UDP)** - - Enter some name, e.g. `fediversity` - - Click Download - - - Write the WireGuard configuration to a file - `fediversity-vpn.config` next to your NixOS configuration - - - Add that file's path to `.git/info/exclude` and make sure - it doesn't otherwise leak (for example, use - [Agenix](https://github.com/ryantm/agenix) to manage - secrets) - - - To your NixOS configuration, add - - ``` nix - networking.wg-quick.interfaces.fediversity.configFile = toString ./fediversity-vpn.config; - ``` -- Select "Promox VE authentication server". -- Ignore the "You do not have a valid subscription" message. - -## Automatically - -This directory contains scripts that can automatically provision or -remove a Proxmox VM. For now, they are tied to one node in the -Fediversity Proxmox, but it would not be difficult to make them more -generic. Try: - -```sh -bash proxmox/provision.sh --help -bash proxmox/remove.sh --help -``` - -## Preparing the machine configuration - -- It is nicer if the machine is a QEMU guest. On NixOS: - - ``` nix - services.qemuGuest.enable = true - ``` - -- Choose name for your machine. - -- Choose static IPs for your machine. The IPv4 and IPv6 subnets - available for Fediversity testing are: - - - `95.215.187.0/24`. Gateway is `95.215.187.1`. - - `2a00:51c0:13:1305::/64`. Gateway is `2a00:51c0:13:1305::1`. - -- I have been using id `XXX` (starting from `001`), name `fediXXX`, - `95.215.187.XXX` and `2a00:51c0:13:1305::XXX`. - -- Name servers should be `95.215.185.6` and `95.215.185.7`. - -- Check [Netbox](https://netbox.protagio.org) to see which addresses - are free. - -## Manually via the GUI - -### Upload your ISO - -- Go to Fediversity proxmox. -- In the left view, expand under the node that you want and click on - "local". -- Select "ISO Images", then click "Upload". -- Note: You can also download from URL. -- Note: You should click on "local" and not "local-zfs". - -### Creating the VM - -- Click "Create VM" at the top right corner. - -#### General - -Node -: which node will host the VM; has to be the same - -VM ID -: Has to be unique, probably best to use the `xxxx` in `vm0xxxx` - (yet to be decided) - -Name -: Usually `vm` + 5 digits, e.g. `vm02199` - -Resource pool -: Fediversity - -#### OS - -Use CD/DVD disc image file (iso) - -: - - Storage - : local, means storage of the node. - - ISO image - : select the image previously uploaded - -No need to touch anything else - -#### System - -BIOS -: OVMF (UEFI) - -EFI Storage -: `linstor_storage`; this is a storage shared by all of the Proxmox - machines. - -Pre-Enroll keys -: MUST be unchecked - -Qemu Agent -: check - -#### Disks - -- Tick "advanced" at the bottom. -- Disk size (GiB) :: 40 (depending on requirements) -- SSD emulation :: check (only visible if "Advanced" is checked) -- Discard :: check, so that blocks of removed data are cleared - -#### CPU - -Sockets -: 1 (depending on requirements) - -Cores -: 2 (depending on requirements) - -Enable NUMA -: check - -#### Memory - -Memory (MiB) -: choose what you want - -Ballooning Device -: leave checked (only visible if "Advanced" is checked) - -#### Network - -Bridge -: `vnet1306`. This is the provisioning bridge; - we will change it later. - -Firewall -: uncheck, we will handle the firewall on the VM itself - -#### Confirm - -### Install and start the VM - -- Start the VM a first time. - - Select the VM in the left panel. You might have to expand the - node on which it is hosted. - - Select "Console" and start the VM. -- Install the VM as you would any other machine. -- [*Shutdown the VM*]{.spurious-link target="Shutdown the VM"}. -- After the VM has been installed: - - Select the VM again, then go to "Hardware". - - Double click on the CD/DVD Drive line. Select "Do not use any - media" and press OK. - - Double click on Network Device, and change the bridge to - `vnet1305`, the public bridge. -- Start the VM again. - -### Remove the VM - -- [*Shutdown the VM*]{.spurious-link target="Shutdown the VM"}. -- On the top right corner, click "More", then "Remove". -- Enter the ID of the machine. -- Check "Purge from job configurations" -- Check "Destroy unreferenced disks owned by guest" -- Click "Remove". - -### Move the VM to another node - -- Make sure there is no ISO plugged in. -- Click on the VM. Click migrate. Choose target node. Go. -- Since the storage is shared, it should go pretty fast (~1 minute). - -### Shutdown the VM - -- Find the VM in the left panel. -- At the top right corner appears a "Shutdown" button with a submenu. -- Clicking "Shutdown" sends a signal to shutdown the machine. This - might not work if the machine is not listening for that signal. -- Brutal solution: in the submenu, select "Stop". -- The checkbox "Overrule active shutdown tasks" means that the machine - should be stopped even if a shutdown is currently ongoing. This is - particularly important if you have tried to shut the machine down - normally just before. diff --git a/infra/operator/garage.nix b/infra/operator/garage.nix new file mode 100644 index 00000000..1f00644b --- /dev/null +++ b/infra/operator/garage.nix @@ -0,0 +1,34 @@ +{ pkgs, ... }: +let + ## NOTE: All of these secrets are publicly available in this source file + ## and will end up in the Nix store. We don't care as they are only ever + ## used for testing anyway. + ## + ## FIXME: Generate and store in state. + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +{ + fediversity = { + garage.enable = true; + pixelfed = pixelfedS3KeyConfig { inherit pkgs; }; + mastodon = mastodonS3KeyConfig { inherit pkgs; }; + peertube = peertubeS3KeyConfig { inherit pkgs; }; + }; +} diff --git a/infra/operator/main.tf b/infra/operator/main.tf new file mode 100644 index 00000000..a72ce35f --- /dev/null +++ b/infra/operator/main.tf @@ -0,0 +1,90 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "= 0.76.1" + } + } +} + +provider "proxmox" { + endpoint = "https://192.168.51.81:8006/" + # because self-signed TLS certificate is in use + insecure = true + + ssh { + agent = true + # TODO: uncomment and configure if using api_token instead of password + username = "root" # FIXME: #24 + } +} + +locals { + # user-facing applications + application_configs = { + # FIXME: wrap applications at the interface to grab them in one go? + mastodon = var.mastodon + pixelfed = var.pixelfed + peertube = var.peertube + } + # services shared between applications + peripherals = { for name in [ + "garage" + ] : name => { + cfg = { + # enable if any user applications are enabled + enable = anytrue([for _, app in local.application_configs: try(app.enable, false)]) + } + } + } +} + +module "nixos" { + source = "../sync-nix" + category = "operator" + + description = each.key + config_nix = each.value.config_nix + config_tf = each.value.config_tf + + # FIXME recheck what may be moved back to sync-nix + for_each = {for name, inst in merge( + local.peripherals, + local.application_configs, + ) : name => merge(inst, { + config_tf = { + fediversityVm = { + name = name # used in hostname, selecting secrets + domain = var.domain + } + fediversity = { + domain = var.domain + temp = { + initialUser = var.initialUser + } + } + } + config_nix = <<-EOF + { + # note interpolations here are TF ones + imports = [ + # shared NixOS config + ${path.root}/../common/shared.nix + # FIXME: separate template options by service + ${path.root}/options.nix + # for service `mastodon` import `mastodon.nix` + # FIXME: get VM details from TF + ${path.root}/../../machines/operator/${inst.hostname} + ${path.module}/${name}.nix + ]; + ## FIXME: switch root authentication to users with password-less sudo, see #24 + users.users.root.openssh.authorizedKeys.keys = let + keys = import ../../keys; + in [ + # allow our panel vm access to the test machines + keys.panel + ]; + } + EOF + }) if try(inst.enable, false)} +} diff --git a/infra/operator/mastodon.nix b/infra/operator/mastodon.nix new file mode 100644 index 00000000..6533442e --- /dev/null +++ b/infra/operator/mastodon.nix @@ -0,0 +1,17 @@ +{ pkgs, ... }: +let + mastodonS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34"; + }; +in +{ + fediversity = { + mastodon = mastodonS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + temp.cores = 1; # FIXME: should come from TF eventually + }; +} diff --git a/infra/operator/options.nix b/infra/operator/options.nix new file mode 100644 index 00000000..446d5da1 --- /dev/null +++ b/infra/operator/options.nix @@ -0,0 +1,55 @@ +# nix options expected to be set from TF here +# TODO: could (part of) this be generated somehow? c.f #275 +{ + lib, + ... +}: +let + inherit (lib) types mkOption; + inherit (types) str enum submodule; +in +{ + options.terraform = { + domain = mkOption { + type = enum [ + "fediversity.net" + ]; + description = '' + Apex domain under which the services will be deployed. + ''; + default = "fediversity.net"; + }; + hostname = mkOption { + type = str; + description = '' + Internal name of the host, e.g. test01 + ''; + }; + initialUser = mkOption { + description = '' + Some services require an initial user to access them. + This option sets the credentials for such an initial user. + ''; + type = submodule { + options = { + displayName = mkOption { + type = str; + description = "Display name of the user"; + }; + username = mkOption { + type = str; + description = "Username for login"; + }; + email = mkOption { + type = str; + description = "User's email address"; + }; + password = mkOption { + type = str; + description = "Password for login"; + }; + }; + }; + }; + }; +} diff --git a/infra/operator/peertube.nix b/infra/operator/peertube.nix new file mode 100644 index 00000000..5f523942 --- /dev/null +++ b/infra/operator/peertube.nix @@ -0,0 +1,20 @@ +{ pkgs, ... }: +let + peertubeS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395"; + }; +in +{ + fediversity = { + peertube = peertubeS3KeyConfig { inherit pkgs; } // { + enable = true; + ## NOTE: Only ever used for testing anyway. + ## + ## FIXME: Generate and store in state. + secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24"; + }; + }; +} diff --git a/infra/operator/pixelfed.nix b/infra/operator/pixelfed.nix new file mode 100644 index 00000000..75790409 --- /dev/null +++ b/infra/operator/pixelfed.nix @@ -0,0 +1,16 @@ +{ pkgs, ... }: +let + pixelfedS3KeyConfig = + { pkgs, ... }: + { + s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b"; + s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987"; + }; +in +{ + fediversity = { + pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // { + enable = true; + }; + }; +} diff --git a/infra/operator/variables.tf b/infra/operator/variables.tf new file mode 100644 index 00000000..05f1b2f6 --- /dev/null +++ b/infra/operator/variables.tf @@ -0,0 +1,51 @@ +# TODO: (partially) generate, say from nix modules, c.f. #275 + +variable "domain" { + type = string + default = "fediversity.net" +} + +variable "mastodon" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "pixelfed" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "peertube" { + type = object({ + enable = bool + }) + default = { + enable = false + } +} + +variable "initialUser" { + type = object({ + displayName = string + username = string + email = string + # TODO: mark (nested) credentials as sensitive + # https://discuss.hashicorp.com/t/is-it-possible-to-mark-an-attribute-of-an-object-as-sensitive/24649/2 + password = string + }) + # FIXME: remove default when the form provides this value, see #285 + default = { + displayName = "Testy McTestface" + username = "test" + email = "test@test.com" + password = "testtest" + } +} diff --git a/infra/pass-ssh-key.sh b/infra/pass-ssh-key.sh new file mode 100755 index 00000000..80f17dec --- /dev/null +++ b/infra/pass-ssh-key.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +export host="$host" + +mkdir -p etc/ssh + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +for keyname in ssh_host_ed25519_key ssh_host_ed25519_key.pub; do + if [[ $keyname == *.pub ]]; then + umask 0133 + else + umask 0177 + fi + cp "$SCRIPT_DIR/../infra/test-machines/${host}/${keyname}" ./etc/ssh/${keyname} +done diff --git a/infra/proxmox-remove.sh b/infra/proxmox-remove.sh deleted file mode 100755 index a8ee2de9..00000000 --- a/infra/proxmox-remove.sh +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env bash -set -euC - -################################################################################ -## Constants - -## FIXME: There seems to be a problem with file upload where the task is -## registered to `node051` no matter what node we are actually uploading to? For -## now, let us just use `node051` everywhere. -readonly node=node051 - -readonly tmpdir=/tmp/proxmox-remove-$RANDOM -mkdir $tmpdir - -################################################################################ -## Parse arguments - -api_url= -username= -password= -vm_ids_or_names= - -help () { - cat </dev/null; do sleep 1; done -} -release_lock () { - rmdir "$tmpdir/lock-$1" -} - -proxmox () { - acquire_lock proxmox - http \ - --verify no \ - --form \ - "$@" \ - "Cookie:PVEAuthCookie=$ticket" \ - "CSRFPreventionToken:$csrf_token" - release_lock proxmox -} - -## Way to inject different behaviour on unexpected status. -# shellcheck disable=SC2317 -default_proxmox_sync_unexpected_status_handler () { - die "unexpected status: '%s'" "$1" -} -proxmox_sync_unexpected_status_handler=default_proxmox_sync_unexpected_status_handler - -## Synchronous variant for when the `proxmox` function would just respond an -## UPID in the `data` JSON field. -proxmox_sync () { - local response upid status - - response=$(proxmox "$@") - upid=$(echo "$response" | jq -r .data) - - while :; do - response=$(proxmox GET "$api_url/nodes/$node/tasks/$upid/status") - status=$(echo "$response" | jq -r .data.status) - - case $status in - running) sleep 1 ;; - stopped) break ;; - *) "$proxmox_sync_unexpected_status_handler" "$status" ;; - esac - done -} - -################################################################################ -## Grab VM options -## -## Takes the name of the VM, grabs `.#vmOptions.` and gets the id from it. - -is_integer () { - [ "$1" -eq "$1" ] 2>/dev/null -} - -grab_vm_options () { - local options - - if is_integer "$1"; then - vm_id=$1 - vm_name="#$1" - - else - vm_name=$1 - - printf 'Grabing VM options for VM %s...\n' "$vm_name" - - options=$( - nix eval \ - --impure --raw --expr " - builtins.toJSON (builtins.getFlake (builtins.toString ./.)).vmOptions.$vm_name - " \ - --log-format raw --quiet - ) - - proxmox=$(echo "$options" | jq -r .proxmox) - vm_id=$(echo "$options" | jq -r .vmId) - - if [ "$proxmox" != fediversity ]; then - die "I do not know how to remove things that are not Fediversity VMs, - but I got proxmox = '%s' for VM %s." "$proxmox" "$vm_name" - fi - - printf 'done grabing VM options for VM %s. Found VM %d on %s Proxmox.\n' \ - "$vm_name" "$vm_id" "$proxmox" - fi -} - -################################################################################ -## Stop VM - -stop_vm () { - printf 'Stopping VM %s...\n' "$vm_name" - - proxmox_sync POST "$api_url/nodes/$node/qemu/$vm_id/status/stop" \ - 'overrule-shutdown'==1 - - printf 'done stopping VM %s.\n' "$vm_name" -} - -################################################################################ -## Delete VM - -# shellcheck disable=SC2317 -proxmox_sync_unexpected_status_handler_ignore_null () { - case $1 in - null) - printf "Attempted to delete VM %s, but got 'null' status. Maybe the VM already does not exist?\n" \ - "$vm_name" - exit 0 - ;; - *) - default_proxmox_sync_unexpected_status_handler "$1" - ;; - esac -} - -delete_vm () { - printf 'Deleting VM %s...\n' "$vm_name" - - proxmox_sync_unexpected_status_handler=proxmox_sync_unexpected_status_handler_ignore_null - proxmox_sync DELETE "$api_url/nodes/$node/qemu/$vm_id" \ - 'destroy-unreferenced-disks'==1 \ - 'purge'==1 - proxmox_sync_unexpected_status_handler=default_proxmox_sync_unexpected_status_handler - - printf 'done deleting VM %s.\n' "$vm_name" -} - -################################################################################ -## Main loop - -printf 'Removing VMs%s...\n' "$vm_ids_or_names" - -remove_vm () ( - grab_vm_options "$1" - stop_vm - delete_vm -) - -for vm_id_or_name in $vm_ids_or_names; do - remove_vm "$vm_id_or_name" & -done - -nb_errors=0 -while :; do - wait -n && : - case $? in - 0) ;; - 127) break ;; - *) nb_errors=$((nb_errors + 1)) ;; - esac -done -if [ "$nb_errors" != 0 ]; then - die 'encountered %d errors while removing VMs%s.' "$nb_errors" "$vm_ids_or_names" -fi - -printf 'done removing VMs%s.\n' "$vm_ids_or_names" - -################################################################################ -## Cleanup - -rm -Rf $tmpdir -exit 0 diff --git a/infra/setup.nix b/infra/setup.nix new file mode 100644 index 00000000..8cefb9f6 --- /dev/null +++ b/infra/setup.nix @@ -0,0 +1,20 @@ +{ + pkgs, + lib, + sources, + ... +}: +pkgs.writeScriptBin "setup" '' + # calculated pins + echo '${lib.strings.toJSON sources}' > sync-nix/.npins.json + # generate TF lock for nix's TF providers + for category in dev operator sync-nix; do + pushd "$category" + rm -rf .terraform/ + rm -f .terraform.lock.hcl + # suppress warning on architecture-specific generated lock file: + # `Warning: Incomplete lock file information for providers`. + tofu init -input=false 1>/dev/null + popd + done +'' diff --git a/infra/shell.nix b/infra/shell.nix new file mode 100644 index 00000000..a6bdf202 --- /dev/null +++ b/infra/shell.nix @@ -0,0 +1 @@ +(import ./. { }).shell diff --git a/infra/sync-nix/main.tf b/infra/sync-nix/main.tf new file mode 100644 index 00000000..390d5bc9 --- /dev/null +++ b/infra/sync-nix/main.tf @@ -0,0 +1,224 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "= 0.76.1" + } + } +} + +locals { + system = "x86_64-linux" + node_name = "node051" + dump_name = "vzdump-qemu-nixos-fediversity-${var.category}.vma.zst" + # dependency paths pre-calculated from npins + pins = jsondecode(file("${path.module}/.npins.json")) + # nix path: expose pins, use nixpkgs in flake commands (`nix run`) + nix_path = "${join(":", [for name, dir in local.pins : "${name}=${dir}"])}:flake=${local.pins["nixpkgs"]}:flake" + config_tf = merge(var.config_tf, { + }) +} + +data "external" "base-hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ${path.module}/../common/nixos/base.nix)\\\"}\""] +} + +# hash of our code directory, used to trigger re-deploy +# FIXME calculate separately to reduce false positives +data "external" "hash" { + program = ["sh", "-c", "echo \"{\\\"hash\\\":\\\"$(nix-hash ..)\\\"}\""] +} + +resource "terraform_data" "template" { + triggers_replace = [ + data.external.base-hash.result, + ] + + provisioner "local-exec" { + working_dir = path.root + environment = { + NIX_PATH = local.nix_path + } + command = <<-EOF + set -euo pipefail + + nixos-generate -f proxmox -o /tmp/nixos-image + ln -s /tmp/nixos-image/vzdump-qemu-nixos-*.vma.zst /tmp/nixos-image/${local.dump_name} + EOF + } +} + +resource "proxmox_virtual_environment_file" "upload" { + lifecycle { + replace_triggered_by = [ + terraform_data.template, + ] + } + + content_type = "images" + datastore_id = "local" + node_name = local.node_name + overwrite = true + + source_file { + path = "/tmp/nixos-image/${local.dump_name}" + file_name = local.dump_name + } +} + +data "proxmox_virtual_environment_vms" "nixos_base" { + node_name = local.node_name + filter { + name = "template" + values = [true] + } +} + +resource "proxmox_virtual_environment_vm" "nix_vm" { + lifecycle { + replace_triggered_by = [ + proxmox_virtual_environment_file.upload, + ] + } + + node_name = local.node_name + pool_id = "Fediversity" + description = var.description + started = true + + agent { + enabled = true + } + + cpu { + type = "x86-64-v2-AES" + cores = var.cores + sockets = var.sockets + numa = true + } + + memory { + dedicated = var.memory + } + + efi_disk { + datastore_id = "linstor_storage" + type = "4m" + } + + disk { + datastore_id = "linstor_storage" + interface = "scsi0" + discard = "on" + iothread = true + size = var.disk_size + ssd = true + } + + clone { + datastore_id = "local" + node_name = data.proxmox_virtual_environment_vms.nixos_base.vms[0].node_name + vm_id = data.proxmox_virtual_environment_vms.nixos_base.vms[0].vm_id + full = true + } + + network_device { + model = "virtio" + bridge = "vnet1306" + } + + operating_system { + type = "l26" + } + + scsi_hardware = "virtio-scsi-single" + bios = "ovmf" +} + +# TF resource to build and deploy NixOS instances. +resource "terraform_data" "nixos" { + + # trigger rebuild/deploy if (FIXME?) any potentially used config/code changed, + # preventing these (20+s, build being bottleneck) when nothing changed. + # terraform-nixos separates these to only deploy if instantiate changed, + # yet building even then - which may be not as bad using deploy on remote. + # having build/deploy one resource reflects wanting to prevent no-op rebuilds + # over preventing (with less false positives) no-op deployments, + # as i could not find a way to do prevent no-op rebuilds without merging them: + # - generic resources cannot have outputs, while we want info from the instantiation (unless built on host?). + # - `data` always runs, which is slow for deploy and especially build. + triggers_replace = [ + data.external.hash.result, + var.config_nix, + var.config_tf, + ] + + provisioner "local-exec" { + # directory to run the script from. we use the TF project root dir, + # here as a path relative from where TF is run from, + # matching calling modules' expectations on config_nix locations. + # note that absolute paths can cause false positives in triggers, + # so are generally discouraged in TF. + working_dir = path.root + environment = { + # nix path used on build, lets us refer to e.g. nixpkgs like `` + NIX_PATH = local.nix_path + } + # TODO: refactor back to command="ignoreme" interpreter=concat([]) to protect sensitive data from error logs? + # TODO: build on target? + command = <<-EOF + set -euo pipefail + + # INSTANTIATE + command=( + nix-instantiate + --expr + 'let + os = import { + system = "${local.system}"; + configuration = + ${var.config_nix} // + # template parameters passed in from TF thru json + builtins.fromJSON "${replace(jsonencode(var.config_tf), "\"", "\\\"")}" // + { + # nix path for debugging + nix.nixPath = [ "${local.nix_path}" ]; + }; + }; + in + # info we want to get back out + { + substituters = builtins.concatStringsSep " " os.config.nix.settings.substituters; + trusted_public_keys = builtins.concatStringsSep " " os.config.nix.settings.trusted-public-keys; + drv_path = os.config.system.build.toplevel.drvPath; + out_path = os.config.system.build.toplevel; + }' + ) + # instantiate the config in /nix/store + "$${command[@]}" -A out_path + # get the other info + json="$("$${command[@]}" --eval --strict --json)" + + # DEPLOY + declare substituters trusted_public_keys drv_path + # set our variables using the json object + eval "export $(echo $json | jaq -r 'to_entries | map("\(.key)=\(.value)") | @sh')" + host="root@${var.hostname}.${var.vm_domain}" # FIXME: #24 + buildArgs=( + --option extra-binary-caches https://cache.nixos.org/ + --option substituters $substituters + --option trusted-public-keys $trusted_public_keys + ) + sshOpts=( + -o BatchMode=yes + -o StrictHostKeyChecking=no + ) + # get the realized derivation to deploy + outPath=$(nix-store --realize "$drv_path" "$${buildArgs[@]}") + # deploy the config by nix-copy-closure + NIX_SSHOPTS="$${sshOpts[*]}" nix-copy-closure --to "$host" "$outPath" --gzip --use-substitutes + # switch the remote host to the config + ssh "$${sshOpts[@]}" "$host" "nix-env --profile /nix/var/nix/profiles/system --set $outPath; $outPath/bin/switch-to-configuration switch" + EOF + } +} diff --git a/infra/sync-nix/variables.tf b/infra/sync-nix/variables.tf new file mode 100644 index 00000000..1bb32b06 --- /dev/null +++ b/infra/sync-nix/variables.tf @@ -0,0 +1,43 @@ +variable "category" { + type = string + description = "Category to be used in naming the base image." +} + +variable "description" { + type = string + default = "" +} + +variable "sockets" { + type = number + description = "The number of sockets of the VM." + default = 1 +} + +variable "cores" { + type = number + description = "The number of cores of the VM." + default = 1 +} + +variable "memory" { + type = number + description = "The amount of memory of the VM in MiB." + default = 2048 +} + +variable "disk_size" { + type = number + description = "The amount of disk of the VM in GiB." + default = 32 +} + +variable "config_nix" { + type = string + default = "{}" +} + +variable "config_tf" { + type = any # map(any): all map elements must have the same type + default = {} +} diff --git a/infra/test-machines/test01/default.nix b/infra/test-machines/test01/default.nix deleted file mode 100644 index 434d7ea5..00000000 --- a/infra/test-machines/test01/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7001; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.51"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::51"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test02/default.nix b/infra/test-machines/test02/default.nix deleted file mode 100644 index a800de18..00000000 --- a/infra/test-machines/test02/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7002; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.52"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::52"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test03/default.nix b/infra/test-machines/test03/default.nix deleted file mode 100644 index e6a31c87..00000000 --- a/infra/test-machines/test03/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7003; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.53"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::53"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test04/default.nix b/infra/test-machines/test04/default.nix deleted file mode 100644 index e7c3b991..00000000 --- a/infra/test-machines/test04/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7004; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.54"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::54"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test05/default.nix b/infra/test-machines/test05/default.nix deleted file mode 100644 index 8b6de173..00000000 --- a/infra/test-machines/test05/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7005; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.55"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::55"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test06/default.nix b/infra/test-machines/test06/default.nix deleted file mode 100644 index f23a83f8..00000000 --- a/infra/test-machines/test06/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7006; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.56"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::56"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test11/default.nix b/infra/test-machines/test11/default.nix deleted file mode 100644 index 3f3de58c..00000000 --- a/infra/test-machines/test11/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7011; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.61"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::61"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test12/default.nix b/infra/test-machines/test12/default.nix deleted file mode 100644 index dcc8530f..00000000 --- a/infra/test-machines/test12/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7012; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.62"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::62"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test13/default.nix b/infra/test-machines/test13/default.nix deleted file mode 100644 index 4a4d7b3d..00000000 --- a/infra/test-machines/test13/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7013; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.63"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::63"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/test-machines/test14/default.nix b/infra/test-machines/test14/default.nix deleted file mode 100644 index ed8ccc46..00000000 --- a/infra/test-machines/test14/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - fediversityVm = { - vmId = 7014; - proxmox = "fediversity"; - - hostPublicKey = builtins.readFile ./ssh_host_ed25519_key.pub; - unsafeHostPrivateKey = builtins.readFile ./ssh_host_ed25519_key; - - domain = "abundos.eu"; - ipv4 = { - address = "95.215.187.64"; - gateway = "95.215.187.1"; - }; - ipv6 = { - address = "2a00:51c0:13:1305::64"; - gateway = "2a00:51c0:13:1305::1"; - }; - }; -} diff --git a/infra/tests.nix b/infra/tests.nix new file mode 100644 index 00000000..2445d1e1 --- /dev/null +++ b/infra/tests.nix @@ -0,0 +1,37 @@ +{ lib, pkgs }: +let + defaults = { + virtualisation = { + memorySize = 2048; + cores = 2; + }; + }; + tf = pkgs.callPackage ./tf.nix { + inherit lib pkgs; + }; + tfEnv = pkgs.callPackage ./tf-env.nix { }; + nodes = { + server = { + environment.systemPackages = [ + tf + tfEnv + ]; + }; + }; +in +lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name; })) { + tf-validate-dev = { + inherit defaults nodes; + testScript = '' + server.wait_for_unit("multi-user.target") + server.succeed("${lib.getExe tf} -chdir='${tfEnv}/infra/dev' validate") + ''; + }; + tf-validate-operator = { + inherit defaults nodes; + testScript = '' + server.wait_for_unit("multi-user.target") + server.succeed("${lib.getExe tf} -chdir='${tfEnv}/infra/operator' validate") + ''; + }; +} diff --git a/infra/tf-env.nix b/infra/tf-env.nix new file mode 100644 index 00000000..78501323 --- /dev/null +++ b/infra/tf-env.nix @@ -0,0 +1,32 @@ +{ + lib, + pkgs, + sources ? import ../npins, + ... +}: +pkgs.stdenv.mkDerivation { + name = "tf-repo"; + src = + with lib.fileset; + toSource { + root = ../.; + # don't copy ignored files + fileset = intersection (gitTracked ../.) ../.; + }; + buildInputs = [ + (import ./tf.nix { inherit lib pkgs; }) + (import ./setup.nix { inherit lib pkgs sources; }) + ]; + buildPhase = '' + runHook preBuild + pushd infra + setup + popd + runHook postBuild + ''; + installPhase = '' + runHook preInstall + cp -r . $out + runHook postInstall + ''; +} diff --git a/infra/tf.nix b/infra/tf.nix new file mode 100644 index 00000000..ba091a34 --- /dev/null +++ b/infra/tf.nix @@ -0,0 +1,34 @@ +# FIXME: use overlays so this gets imported just once? +{ + lib, + pkgs, + ... +}: +let + tofuProvider = + provider: + provider.override (oldArgs: { + provider-source-address = + lib.replaceStrings [ "https://registry.terraform.io/providers" ] [ "registry.opentofu.org" ] + oldArgs.homepage; + }); + tf = pkgs.opentofu; + tfPlugins = ( + p: [ + p.external + (pkgs.terraform-providers.mkProvider { + owner = "bpg"; + repo = "terraform-provider-proxmox"; + rev = "v0.76.1"; + spdx = "MPL-2.0"; + hash = "sha256-SgaqMcvf1vhWqzWLBtj35yuyRch3m2/sWy15sQzo0ck="; + vendorHash = "sha256-xNnpIH5UTWS9otUi1+ttnxK039DvsbolOni75ver1YA="; + homepage = "https://registry.terraform.io/providers/bpg/proxmox"; + provider-source-address = "registry.opentofu.org/bpg/proxmox"; + }) + ] + ); +in +# tf.withPlugins tfPlugins +# https://github.com/NixOS/nixpkgs/pull/358522 +tf.withPlugins (p: pkgs.lib.lists.map tofuProvider (tfPlugins p)) diff --git a/keys/README.md b/keys/README.md index 0de57810..b562a3ca 100644 --- a/keys/README.md +++ b/keys/README.md @@ -14,7 +14,7 @@ overwrite a secret without knowing its contents.) In infra management, the systems' keys are used for security reasons; they identify the machine that we are talking to. The contributor keys are used to give access to the `root` user on these machines, which allows, among other -things, to deploy their configurations with NixOps4. +things, to deploy their configurations. ## Adding a contributor diff --git a/machines/README.md b/machines/README.md new file mode 100644 index 00000000..d51df126 --- /dev/null +++ b/machines/README.md @@ -0,0 +1,4 @@ +# Machines + +This directory contains the definition of [the VMs](machines.md) that host our +infrastructure. diff --git a/infra/machines/fedi200/default.nix b/machines/dev/fedi200/default.nix similarity index 100% rename from infra/machines/fedi200/default.nix rename to machines/dev/fedi200/default.nix diff --git a/machines/dev/fedi200/dns.nix b/machines/dev/fedi200/dns.nix new file mode 100644 index 00000000..87a13d7f --- /dev/null +++ b/machines/dev/fedi200/dns.nix @@ -0,0 +1,2 @@ +_: { +} diff --git a/infra/machines/fedi201/default.nix b/machines/dev/fedi201/default.nix similarity index 82% rename from infra/machines/fedi201/default.nix rename to machines/dev/fedi201/default.nix index 2f8978d7..c8567117 100644 --- a/infra/machines/fedi201/default.nix +++ b/machines/dev/fedi201/default.nix @@ -14,10 +14,4 @@ gateway = "2a00:51c0:13:1305::1"; }; }; - - nixos.module = { - imports = [ - ./fedipanel.nix - ]; - }; } diff --git a/infra/machines/fedi201/fedipanel.nix b/machines/dev/fedi201/fedipanel.nix similarity index 58% rename from infra/machines/fedi201/fedipanel.nix rename to machines/dev/fedi201/fedipanel.nix index 5c4236fc..fdc8205c 100644 --- a/infra/machines/fedi201/fedipanel.nix +++ b/machines/dev/fedi201/fedipanel.nix @@ -7,12 +7,12 @@ let in { imports = [ + (import ../../../panel { }).module ]; security.acme = { acceptTerms = true; - defaults.email = "beheer@procolix.com"; }; age.secrets.panel-ssh-key = { @@ -37,6 +37,24 @@ in enable = true; production = true; domain = "demo.fediversity.eu"; + # FIXME: make it work without this duplication + settings = + let + cfg = config.services.${name}; + in + { + STATIC_ROOT = "/var/lib/${name}/static"; + DEBUG = false; + ALLOWED_HOSTS = [ + cfg.domain + cfg.host + "localhost" + "[::1]" + ]; + CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ]; + COMPRESS_OFFLINE = true; + LIBSASS_OUTPUT_STYLE = "compressed"; + }; secrets = { SECRET_KEY = config.age.secrets.panel-secret-key.path; }; diff --git a/machines/dev/vm02116/default.nix b/machines/dev/vm02116/default.nix new file mode 100644 index 00000000..b92faab4 --- /dev/null +++ b/machines/dev/vm02116/default.nix @@ -0,0 +1,31 @@ +{ lib, ... }: +{ + fediversityVm = { + vmId = 2116; + proxmox = "procolix"; + description = "Forgejo"; + + ipv4.address = "185.206.232.34"; + ipv6.address = "2a00:51c0:12:1201::20"; + }; + + ## vm02116 is running on old hardware based on a Xen VM environment, so it + ## needs these extra options. Once the VM gets moved to a newer node, these + ## two options can safely be removed. + boot.initrd.availableKernelModules = [ "xen_blkfront" ]; + services.xe-guest-utilities.enable = true; + + ## NOTE: This VM was created manually, which requires us to override the + ## default disko-based `fileSystems` definition. + fileSystems = lib.mkForce { + "/" = { + device = "/dev/disk/by-uuid/3802a66d-e31a-4650-86f3-b51b11918853"; + fsType = "ext4"; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/2CE2-1173"; + fsType = "vfat"; + }; + }; +} diff --git a/infra/machines/vm02116/forgejo.nix b/machines/dev/vm02116/forgejo.nix similarity index 100% rename from infra/machines/vm02116/forgejo.nix rename to machines/dev/vm02116/forgejo.nix diff --git a/machines/dev/vm02187/default.nix b/machines/dev/vm02187/default.nix new file mode 100644 index 00000000..0017e0ef --- /dev/null +++ b/machines/dev/vm02187/default.nix @@ -0,0 +1,29 @@ +{ lib, ... }: +{ + fediversityVm = { + vmId = 2187; + proxmox = "procolix"; + description = "Wiki"; + + ipv4.address = "185.206.232.187"; + ipv6.address = "2a00:51c0:12:1201::187"; + }; + + ## NOTE: This VM was created manually, which requires us to override the + ## default disko-based `fileSystems` definition. + fileSystems = lib.mkForce { + "/" = { + device = "/dev/disk/by-uuid/a46a9c46-e32b-4216-a4aa-8819b2cd0d49"; + fsType = "ext4"; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/6AB5-4FA8"; + fsType = "vfat"; + options = [ + "fmask=0022" + "dmask=0022" + ]; + }; + }; +} diff --git a/infra/machines/vm02187/wiki.nix b/machines/dev/vm02187/wiki.nix similarity index 100% rename from infra/machines/vm02187/wiki.nix rename to machines/dev/vm02187/wiki.nix diff --git a/infra/test-machines/configuration.json b/machines/operator/configuration.json similarity index 100% rename from infra/test-machines/configuration.json rename to machines/operator/configuration.json diff --git a/infra/test-machines/test01/ssh_host_ed25519_key b/machines/operator/test01/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test01/ssh_host_ed25519_key rename to machines/operator/test01/ssh_host_ed25519_key diff --git a/infra/test-machines/test01/ssh_host_ed25519_key.pub b/machines/operator/test01/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test01/ssh_host_ed25519_key.pub rename to machines/operator/test01/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test02/ssh_host_ed25519_key b/machines/operator/test02/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test02/ssh_host_ed25519_key rename to machines/operator/test02/ssh_host_ed25519_key diff --git a/infra/test-machines/test02/ssh_host_ed25519_key.pub b/machines/operator/test02/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test02/ssh_host_ed25519_key.pub rename to machines/operator/test02/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test03/ssh_host_ed25519_key b/machines/operator/test03/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test03/ssh_host_ed25519_key rename to machines/operator/test03/ssh_host_ed25519_key diff --git a/infra/test-machines/test03/ssh_host_ed25519_key.pub b/machines/operator/test03/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test03/ssh_host_ed25519_key.pub rename to machines/operator/test03/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test04/ssh_host_ed25519_key b/machines/operator/test04/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test04/ssh_host_ed25519_key rename to machines/operator/test04/ssh_host_ed25519_key diff --git a/infra/test-machines/test04/ssh_host_ed25519_key.pub b/machines/operator/test04/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test04/ssh_host_ed25519_key.pub rename to machines/operator/test04/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test05/ssh_host_ed25519_key b/machines/operator/test05/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test05/ssh_host_ed25519_key rename to machines/operator/test05/ssh_host_ed25519_key diff --git a/infra/test-machines/test05/ssh_host_ed25519_key.pub b/machines/operator/test05/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test05/ssh_host_ed25519_key.pub rename to machines/operator/test05/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test06/ssh_host_ed25519_key b/machines/operator/test06/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test06/ssh_host_ed25519_key rename to machines/operator/test06/ssh_host_ed25519_key diff --git a/infra/test-machines/test06/ssh_host_ed25519_key.pub b/machines/operator/test06/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test06/ssh_host_ed25519_key.pub rename to machines/operator/test06/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test11/ssh_host_ed25519_key b/machines/operator/test11/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test11/ssh_host_ed25519_key rename to machines/operator/test11/ssh_host_ed25519_key diff --git a/infra/test-machines/test11/ssh_host_ed25519_key.pub b/machines/operator/test11/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test11/ssh_host_ed25519_key.pub rename to machines/operator/test11/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test12/ssh_host_ed25519_key b/machines/operator/test12/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test12/ssh_host_ed25519_key rename to machines/operator/test12/ssh_host_ed25519_key diff --git a/infra/test-machines/test12/ssh_host_ed25519_key.pub b/machines/operator/test12/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test12/ssh_host_ed25519_key.pub rename to machines/operator/test12/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test13/ssh_host_ed25519_key b/machines/operator/test13/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test13/ssh_host_ed25519_key rename to machines/operator/test13/ssh_host_ed25519_key diff --git a/infra/test-machines/test13/ssh_host_ed25519_key.pub b/machines/operator/test13/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test13/ssh_host_ed25519_key.pub rename to machines/operator/test13/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test14/ssh_host_ed25519_key b/machines/operator/test14/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test14/ssh_host_ed25519_key rename to machines/operator/test14/ssh_host_ed25519_key diff --git a/infra/test-machines/test14/ssh_host_ed25519_key.pub b/machines/operator/test14/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test14/ssh_host_ed25519_key.pub rename to machines/operator/test14/ssh_host_ed25519_key.pub diff --git a/npins/sources.json b/npins/sources.json index 4971590b..df164880 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -25,18 +25,21 @@ "url": null, "hash": "1w2gsy6qwxa5abkv8clb435237iifndcxq0s79wihqw11a5yb938" }, - "flake-parts": { - "type": "Git", + "disko": { + "type": "GitRelease", "repository": { "type": "GitHub", - "owner": "hercules-ci", - "repo": "flake-parts" + "owner": "nix-community", + "repo": "disko" }, - "branch": "main", + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, "submodules": false, - "revision": "c621e8422220273271f52058f618c94e405bb0f5", - "url": "https://github.com/hercules-ci/flake-parts/archive/c621e8422220273271f52058f618c94e405bb0f5.tar.gz", - "hash": "09j2dafd75ydlcw8v48vcpfm2mw0j6cs8286x2hha2lr08d232w4" + "version": "v1.11.0", + "revision": "cdf8deded8813edfa6e65544f69fdd3a59fa2bb4", + "url": "https://api.github.com/repos/nix-community/disko/tarball/v1.11.0", + "hash": "13brimg7z7k9y36n4jc1pssqyw94nd8qvgfjv53z66lv4xkhin92" }, "git-hooks": { "type": "Git", @@ -64,6 +67,19 @@ "url": "https://github.com/hercules-ci/gitignore.nix/archive/637db329424fd7e46cf4185293b9cc8c88c95394.tar.gz", "hash": "02wxkdpbhlm3yk5mhkhsp3kwakc16xpmsf2baw57nz1dg459qv8w" }, + "home-manager": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "nix-community", + "repo": "home-manager" + }, + "branch": "master", + "submodules": false, + "revision": "22b326b42bf42973d5e4fe1044591fb459e6aeac", + "url": "https://github.com/nix-community/home-manager/archive/22b326b42bf42973d5e4fe1044591fb459e6aeac.tar.gz", + "hash": "0hwllnym5mrrxinjsq0p9zn39i110c1xixp4x64svl7jjm5zb4c4" + }, "htmx": { "type": "GitRelease", "repository": { diff --git a/panel/default.nix b/panel/default.nix index a9c20f84..85ea2afd 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -21,11 +21,18 @@ in pkgs.npins manage ]; - env = import ./env.nix { inherit lib pkgs; } // { - NPINS_DIRECTORY = toString ../npins; - CREDENTIALS_DIRECTORY = toString ./.credentials; - DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; - }; + env = + let + inherit (builtins) toString; + in + import ./env.nix { inherit lib pkgs; } + // { + NPINS_DIRECTORY = toString ../npins; + CREDENTIALS_DIRECTORY = toString ./.credentials; + DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; + # locally: use a fixed relative reference, so we can use our newest files without copying to the store + REPO_DIR = toString ../.; + }; shellHook = '' ${lib.concatStringsSep "\n" ( map (file: "ln -sf ${file.from} ${toString ./src/${file.to}}") package.generated diff --git a/panel/env.nix b/panel/env.nix index 07ce4193..9dfddf1c 100644 --- a/panel/env.nix +++ b/panel/env.nix @@ -3,16 +3,16 @@ pkgs, ... }: -let - inherit (builtins) toString; -in { - REPO_DIR = toString ../.; - # explicitly use nix, as e.g. lix does not have configurable-impure-env BIN_PATH = lib.makeBinPath [ - # explicitly use nix, as e.g. lix does not have configurable-impure-env - pkgs.nix - # nixops error maybe due to our flake git hook: executing 'git': No such file or directory + pkgs.lix + pkgs.bash + pkgs.coreutils + pkgs.openssh pkgs.git + pkgs.direnv + pkgs.jaq # tf + (import ../infra/tf.nix { inherit lib pkgs; }) ]; + SSH_PRIVATE_KEY_FILE = ""; } diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index ecf06e0f..1cb1fdd7 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -29,6 +29,9 @@ let ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) (builtins.toFile "extra-settings.py" cfg.extra-settings) ]; + REPO_DIR = import ../../infra/tf-env.nix { + inherit lib pkgs; + }; }; python-environment = pkgs.python3.withPackages ( @@ -157,9 +160,7 @@ in }; }; - users.users.${name} = { - isNormalUser = true; - }; + users.users.${name}.isNormalUser = true; users.groups.${name} = { }; systemd.services.${name} = { @@ -167,6 +168,7 @@ in after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = [ + pkgs.openssh python-environment manage-service ]; diff --git a/panel/nix/package.nix b/panel/nix/package.nix index e13e015a..a57a0592 100644 --- a/panel/nix/package.nix +++ b/panel/nix/package.nix @@ -1,5 +1,6 @@ { lib, + pkgs, sqlite, python3, python3Packages, @@ -14,7 +15,7 @@ let root = ../src; fileset = intersection (gitTracked ../../.) ../src; }; - pyproject = with lib; fromTOML pyproject-toml; + pyproject = fromTOML pyproject-toml; # TODO: define this globally name = "panel"; # TODO: we may want this in a file so it's easier to read statically @@ -89,7 +90,13 @@ python3.pkgs.buildPythonPackage { mkdir -p $out/bin cp -v ${src}/manage.py $out/bin/manage.py chmod +x $out/bin/manage.py - wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH" + wrapProgram $out/bin/manage.py \ + --set REPO_DIR "${ + import ../../infra/tf-env.nix { + inherit lib pkgs; + } + }" \ + --prefix PYTHONPATH : "$PYTHONPATH" ${lib.concatStringsSep "\n" ( map (file: "cp ${file.from} $out/${python3.sitePackages}/${file.to}") generated )} diff --git a/panel/nix/python-packages/django-pydantic-field/default.nix b/panel/nix/python-packages/django-pydantic-field/default.nix index 2648a0db..74349e3e 100644 --- a/panel/nix/python-packages/django-pydantic-field/default.nix +++ b/panel/nix/python-packages/django-pydantic-field/default.nix @@ -1,3 +1,4 @@ +# TODO upstream, see #248 { lib, buildPythonPackage, diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index bbfa753a..95647897 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -10,14 +10,19 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/4.2/ref/settings/ """ +import re import sys +import subprocess import os +import json import importlib.util import dj_database_url from os import environ as env from pathlib import Path +STORE_PATTERN = re.compile("^/nix/store/[^/]+$") + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -242,6 +247,7 @@ if user_settings_file is not None: # PATH to expose to launch button bin_path=env['BIN_PATH'] -# path of the root flake to trigger nixops from, see #94. +# path of the root flake to deploy from # to deploy this should be specified, for dev just use a relative path. repo_dir = env["REPO_DIR"] + diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index d11f8cd8..d25b53dd 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -1,6 +1,9 @@ from enum import Enum import json +from os.path import expanduser +from pathlib import Path import subprocess +import logging import os from django.urls import reverse_lazy @@ -19,9 +22,7 @@ from pydantic import BaseModel from panel import models, settings from panel.configuration import schema -class Index(TemplateView): - template_name = 'index.html' - +logger = logging.getLogger(__name__) class Index(TemplateView): template_name = 'index.html' @@ -110,8 +111,13 @@ class DeploymentStatus(ConfigurationForm): } env = { "PATH": settings.bin_path, + # needed by direnv + "HOME": str(Path.home()), + # "TF_LOG": "info", + } | { # pass in form info to our deployment - "DEPLOYMENT": config.json() + # FIXME: ensure sensitive info is protected + f"TF_VAR_{k}": v if isinstance(v, str) else json.dumps(v) for k, v in json.loads(config.model_dump_json()).items() } # XXX should we not log this if it may show proxmox credentials from `.envrc`s? # those could instead be passed as sensitive TF vars, but that would not address this. @@ -120,14 +126,21 @@ class DeploymentStatus(ConfigurationForm): # direnv wants this run upfront, and chaining in subprocess feels awkward subprocess.check_call(["direnv", "allow"], cwd=cwd) cmd = [ - "nix", - "develop", - "--extra-experimental-features", - "configurable-impure-env", - "--command", - "nixops4", + # pick up env vars from any .gitignore'd `.envrc`. + # if this might fall back to `infra/.envrc`, which loads a nix environment, + # is this really an elegant approach to pass env vars for our + # different scenarios (deployed vs local panel vs local direct TF invocations)? + "direnv", + "exec", + cwd, + # run TF + "tofu", + # f"-chdir={cwd}", "apply", - "test", + f"-state={cwd}/terraform.tfstate", # FIXME: separate users' state, see #313 + "--auto-approve", + "-lock=false", + "-parallelism=1" # limit OOM risk ] deployment_result = subprocess.run(cmd, cwd=cwd, env=env) logger.debug("deployment_result: %s", deployment_result) diff --git a/infra/makeInstallerIso.nix b/proxmox/makeInstallerIso.nix similarity index 99% rename from infra/makeInstallerIso.nix rename to proxmox/makeInstallerIso.nix index 501acad1..a8156274 100644 --- a/infra/makeInstallerIso.nix +++ b/proxmox/makeInstallerIso.nix @@ -20,7 +20,6 @@ let installer = { - config, pkgs, lib, ... diff --git a/secrets/.envrc b/secrets/.envrc new file mode 100644 index 00000000..c6aca5a1 --- /dev/null +++ b/secrets/.envrc @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +# shellcheck shell=bash +if type -P lorri &>/dev/null; then + eval "$(lorri direnv)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use_nix +fi diff --git a/secrets/README.md b/secrets/README.md index 345bd134..65840925 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -22,7 +22,7 @@ As an example, let us add a secret in a file “cheeses” whose content should extension); this will open your `$EDITOR` ; enter “best ones come unpasteurised”, save and close. -3. If you are doing something flake-related such as NixOps4, remember to commit +3. If you are doing something flake-related, remember to commit or at least stage the secret. 4. In the machine's configuration, load our `ageSecrets` NixOS module, declare the machine's host key and start using your secrets, eg.: diff --git a/secrets/default.nix b/secrets/default.nix new file mode 100644 index 00000000..20d5f5bd --- /dev/null +++ b/secrets/default.nix @@ -0,0 +1,24 @@ +{ + system ? builtins.currentSystem, + sources ? import ../npins, + pkgs ? import sources.nixpkgs { inherit system; }, +}: +let + inherit (sources) agenix; +in +{ + # shell for testing TF directly + shell = pkgs.mkShellNoCC { + packages = [ + (pkgs.callPackage "${agenix}/pkgs/agenix.nix" { }) + ]; + }; + + # re-export inputs so they can be overridden granularly + # (they can't be accessed from the outside any other way) + inherit + sources + system + pkgs + ; +} diff --git a/secrets/shell.nix b/secrets/shell.nix new file mode 100644 index 00000000..a6bdf202 --- /dev/null +++ b/secrets/shell.nix @@ -0,0 +1 @@ +(import ./. { }).shell diff --git a/services/.envrc b/services/.envrc new file mode 100644 index 00000000..c6aca5a1 --- /dev/null +++ b/services/.envrc @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +# shellcheck shell=bash +if type -P lorri &>/dev/null; then + eval "$(lorri direnv)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use_nix +fi diff --git a/services/fediversity/default.nix b/services/fediversity/default.nix index 4f793c03..7e20e0c3 100644 --- a/services/fediversity/default.nix +++ b/services/fediversity/default.nix @@ -31,7 +31,7 @@ in type = types.submodule { options = { cores = mkOption { - description = "number of cores; should be obtained from NixOps4"; + description = "number of cores; should be obtained from TF"; type = types.int; }; diff --git a/services/fediversity/peertube/options.nix b/services/fediversity/peertube/options.nix index 21eaa021..3b99dda1 100644 --- a/services/fediversity/peertube/options.nix +++ b/services/fediversity/peertube/options.nix @@ -20,7 +20,7 @@ in description = '' Internal option — change at your own risk - FIXME: should it be provided by NixOps4? + FIXME: should it be provided by TF? or maybe we should just ask for a main secret from which to derive all the others? ''; };