From 1b832c1f5b4b86c814aa435d04edb328d6b76d6f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 12 Jun 2025 13:05:11 +0200 Subject: [PATCH 01/37] bypass native flake input for Nixpkgs (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @Niols the sheer amount of hassle and noise indicates that it may be better to first split out a `flake.nix` just for the tests. And all this clutter doesn't even explain yet *why* we thought it needs to be there. closes #279. Co-authored-by: Nicolas “Niols” Jeannerod Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/374 Reviewed-by: kiara Grouwstra Co-authored-by: Valentin Gagarin Co-committed-by: Valentin Gagarin --- deployment/check/basic/nixosTest.nix | 6 + deployment/check/common/deployerNode.nix | 4 + flake.lock | 19 +-- flake.nix | 136 ++++++++++-------- npins/sources.json | 16 +++ services/fediversity/pixelfed/default.nix | 6 - .../pixelfed/group-permissions.patch | 18 --- 7 files changed, 106 insertions(+), 99 deletions(-) delete mode 100644 services/fediversity/pixelfed/group-permissions.patch diff --git a/deployment/check/basic/nixosTest.nix b/deployment/check/basic/nixosTest.nix index 600baefb..fe9bda09 100644 --- a/deployment/check/basic/nixosTest.nix +++ b/deployment/check/basic/nixosTest.nix @@ -10,6 +10,12 @@ inputs.nixops4.packages.${pkgs.system}.default ]; + # FIXME: sad times + system.extraDependencies = with pkgs; [ + jq + jq.inputDerivation + ]; + system.extraDependenciesFromModule = { pkgs, ... }: { diff --git a/deployment/check/common/deployerNode.nix b/deployment/check/common/deployerNode.nix index 36f2897d..7236fc26 100644 --- a/deployment/check/common/deployerNode.nix +++ b/deployment/check/common/deployerNode.nix @@ -14,6 +14,8 @@ let types ; + sources = import ../../../npins; + in { imports = [ ./sharedOptions.nix ]; @@ -57,6 +59,8 @@ in "${inputs.nixops4-nixos}" "${inputs.nixpkgs}" + "${sources.flake-inputs}" + pkgs.stdenv pkgs.stdenvNoCC ] diff --git a/flake.lock b/flake.lock index 4eff9508..baf0a2bc 100644 --- a/flake.lock +++ b/flake.lock @@ -596,22 +596,6 @@ "type": "github" } }, - "nixpkgs_4": { - "locked": { - "lastModified": 1740463929, - "narHash": "sha256-4Xhu/3aUdCKeLfdteEHMegx5ooKQvwPHNkOgNCXQrvc=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "5d7db4668d7a0c6cc5fc8cf6ef33b008b2b1ed8b", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixos-24.11", - "repo": "nixpkgs", - "type": "github" - } - }, "parts": { "inputs": { "nixpkgs-lib": [ @@ -686,8 +670,7 @@ "nixops4-nixos", "nixops4" ], - "nixops4-nixos": "nixops4-nixos", - "nixpkgs": "nixpkgs_4" + "nixops4-nixos": "nixops4-nixos" } }, "rust-overlay": { diff --git a/flake.nix b/flake.nix index 6dd3d3df..e5b6a103 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,5 @@ { inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; # consumed by flake-parts flake-parts.url = "github:hercules-ci/flake-parts"; git-hooks.url = "github:cachix/git-hooks.nix"; nixops4.follows = "nixops4-nixos/nixops4"; @@ -8,65 +7,88 @@ }; outputs = - inputs@{ flake-parts, ... }: + inputs@{ self, flake-parts, ... }: let sources = import ./npins; + inherit (import sources.flake-inputs) import-flake; inherit (sources) git-hooks agenix; + # XXX(@fricklerhandwerk): this atrocity is required to splice in a foreign Nixpkgs via flake-parts + # XXX - this is just importing a flake + nixpkgs = import-flake { src = sources.nixpkgs; }; + # XXX - this overrides the inputs attached to `self` + inputs' = self.inputs // { + nixpkgs = nixpkgs; + }; + self' = self // { + inputs = inputs'; + }; in - flake-parts.lib.mkFlake { inherit inputs; } { - systems = [ - "x86_64-linux" - "aarch64-linux" - "x86_64-darwin" - "aarch64-darwin" - ]; - - imports = [ - (import "${git-hooks}/flake-module.nix") - inputs.nixops4.modules.flake.default - - ./deployment/flake-part.nix - ./infra/flake-part.nix - ]; - - perSystem = - { - pkgs, - lib, - inputs', - ... - }: - { - formatter = pkgs.nixfmt-rfc-style; - - pre-commit.settings.hooks = - let - ## Add a directory here if pre-commit hooks shouldn't apply to it. - optout = [ "npins" ]; - excludes = map (dir: "^${dir}/") optout; - addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); - in - addExcludes { - nixfmt-rfc-style.enable = true; - deadnix.enable = true; - trim-trailing-whitespace.enable = true; - shellcheck.enable = true; - }; - - devShells.default = pkgs.mkShell { - packages = [ - pkgs.npins - pkgs.nil - (pkgs.callPackage "${agenix}/pkgs/agenix.nix" { }) - pkgs.openssh - pkgs.httpie - pkgs.jq - # exposing this env var as a hack to pass info in from form - (inputs'.nixops4.packages.default.overrideAttrs { - impureEnvVars = [ "DEPLOYMENT" ]; - }) - ]; - }; + # XXX - finally we override the overall set of `inputs` -- we need both: + # `flake-parts obtains `nixpkgs` from `self.inputs` and not from `inputs`. + flake-parts.lib.mkFlake + { + inputs = inputs // { + inherit nixpkgs; }; - }; + self = self'; + } + ( + { inputs, ... }: + { + systems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + imports = [ + (import "${git-hooks}/flake-module.nix") + inputs.nixops4.modules.flake.default + + ./deployment/flake-part.nix + ./infra/flake-part.nix + ]; + + perSystem = + { + pkgs, + lib, + inputs', + ... + }: + { + formatter = pkgs.nixfmt-rfc-style; + + pre-commit.settings.hooks = + let + ## Add a directory here if pre-commit hooks shouldn't apply to it. + optout = [ "npins" ]; + excludes = map (dir: "^${dir}/") optout; + addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); + in + addExcludes { + nixfmt-rfc-style.enable = true; + deadnix.enable = true; + trim-trailing-whitespace.enable = true; + shellcheck.enable = true; + }; + + devShells.default = pkgs.mkShell { + packages = [ + pkgs.npins + pkgs.nil + (pkgs.callPackage "${agenix}/pkgs/agenix.nix" { }) + pkgs.openssh + pkgs.httpie + pkgs.jq + # exposing this env var as a hack to pass info in from form + (inputs'.nixops4.packages.default.overrideAttrs { + impureEnvVars = [ "DEPLOYMENT" ]; + }) + ]; + }; + }; + } + ); } diff --git a/npins/sources.json b/npins/sources.json index 4971590b..657e8a3a 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -25,6 +25,22 @@ "url": null, "hash": "1w2gsy6qwxa5abkv8clb435237iifndcxq0s79wihqw11a5yb938" }, + "flake-inputs": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "fricklerhandwerk", + "repo": "flake-inputs" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "4.1", + "revision": "ad02792f7543754569fe2fd3d5787ee00ef40be2", + "url": "https://api.github.com/repos/fricklerhandwerk/flake-inputs/tarball/4.1", + "hash": "1j57avx2mqjnhrsgq3xl7ih8v7bdhz1kj3min6364f486ys048bm" + }, "flake-parts": { "type": "Git", "repository": { diff --git a/services/fediversity/pixelfed/default.nix b/services/fediversity/pixelfed/default.nix index d6328e3b..9080e6ba 100644 --- a/services/fediversity/pixelfed/default.nix +++ b/services/fediversity/pixelfed/default.nix @@ -56,12 +56,6 @@ in ) (mkIf config.fediversity.pixelfed.enable { - ## NOTE: Pixelfed as packaged in nixpkgs has a permission issue that prevents Nginx - ## from being able to serving the images. We fix it here, but this should be - ## upstreamed. See https://github.com/NixOS/nixpkgs/issues/235147 - services.pixelfed.package = pkgs.pixelfed.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ ./group-permissions.patch ]; - }); users.users.nginx.extraGroups = [ "pixelfed" ]; services.pixelfed = { diff --git a/services/fediversity/pixelfed/group-permissions.patch b/services/fediversity/pixelfed/group-permissions.patch deleted file mode 100644 index d7dd442d..00000000 --- a/services/fediversity/pixelfed/group-permissions.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/config/filesystems.php b/config/filesystems.php -index 00254e93..fc1a58f3 100644 ---- a/config/filesystems.php -+++ b/config/filesystems.php -@@ -49,11 +49,11 @@ return [ - 'permissions' => [ - 'file' => [ - 'public' => 0644, -- 'private' => 0600, -+ 'private' => 0640, - ], - 'dir' => [ - 'public' => 0755, -- 'private' => 0700, -+ 'private' => 0750, - ], - ], - ], -- 2.48.1 From 5a514b96e9de77d5ff52f553d98341adffe38edf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 6 Jun 2025 12:14:34 +0200 Subject: [PATCH 02/37] use deployed environment for launching nixops4 from the panel --- panel/env.nix | 8 -------- panel/nix/configuration.nix | 4 ++++ panel/src/panel/settings.py | 2 -- panel/src/panel/views.py | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/panel/env.nix b/panel/env.nix index 07ce4193..6006016d 100644 --- a/panel/env.nix +++ b/panel/env.nix @@ -1,6 +1,4 @@ { - lib, - pkgs, ... }: let @@ -9,10 +7,4 @@ in { REPO_DIR = toString ../.; # explicitly use nix, as e.g. lix does not have configurable-impure-env - BIN_PATH = lib.makeBinPath [ - # explicitly use nix, as e.g. lix does not have configurable-impure-env - pkgs.nix - # nixops error maybe due to our flake git hook: executing 'git': No such file or directory - pkgs.git - ]; } diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index b0bcd5c4..9269cc2a 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -181,6 +181,10 @@ in path = [ python-environment manage-service + # XXX(@fricklerhandwerk): NixOps4 needs a Nix available. + pkgs.nix + # TODO(@fricklerhandwerk): Only needed because we invoke NixOps4 via `nix develop`, remove once that's gone. + pkgs.git ]; preStart = '' # Auto-migrate on first run or if the package has changed diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index bbfa753a..14b1a96f 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -240,8 +240,6 @@ if user_settings_file is not None: # The correct thing to do here would be using a helper function such as with `get_secret()` that will catch the exception and explain what's wrong and where to put the right values. # Replacing the `USER_SETTINGS_FILE` mechanism following the comment there would probably be a good thing. -# PATH to expose to launch button -bin_path=env['BIN_PATH'] # path of the root flake to trigger nixops from, see #94. # to deploy this should be specified, for dev just use a relative path. repo_dir = env["REPO_DIR"] diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 9b1e902e..ad0bbf58 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -89,7 +89,7 @@ class DeploymentStatus(ConfigurationForm): def deployment(self, config: BaseModel): env = { - "PATH": settings.bin_path, + "PATH": os.environ.get("PATH"), # pass in form info to our deployment "DEPLOYMENT": config.json() } -- 2.48.1 From dbb4ce67fc784f5040926030bde338a2e2457900 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Sun, 15 Jun 2025 15:01:56 +0200 Subject: [PATCH 03/37] move machines to reflect a semantic structure (#367) later we may want to distinguish dev vs host as well, tho eventually we expect not to have hard-coded machines anyway. split off from #319. Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/367 Co-authored-by: Kiara Grouwstra Co-committed-by: Kiara Grouwstra --- infra/README.md | 4 ++-- infra/flake-part.nix | 8 ++++---- machines/README.md | 4 ++++ {infra/machines => machines/dev}/fedi200/default.nix | 0 {infra/machines => machines/dev}/fedi201/default.nix | 0 {infra/machines => machines/dev}/fedi201/fedipanel.nix | 0 {infra/machines => machines/dev}/vm02116/default.nix | 0 {infra/machines => machines/dev}/vm02116/forgejo.nix | 0 {infra/machines => machines/dev}/vm02187/default.nix | 0 {infra/machines => machines/dev}/vm02187/wiki.nix | 0 {infra => machines}/machines.md | 0 {infra => machines}/machines.md.sh | 2 +- .../operator}/test01/default.nix | 0 .../operator}/test01/ssh_host_ed25519_key | 0 .../operator}/test01/ssh_host_ed25519_key.pub | 0 .../operator}/test02/default.nix | 0 .../operator}/test02/ssh_host_ed25519_key | 0 .../operator}/test02/ssh_host_ed25519_key.pub | 0 .../operator}/test03/default.nix | 0 .../operator}/test03/ssh_host_ed25519_key | 0 .../operator}/test03/ssh_host_ed25519_key.pub | 0 .../operator}/test04/default.nix | 0 .../operator}/test04/ssh_host_ed25519_key | 0 .../operator}/test04/ssh_host_ed25519_key.pub | 0 .../operator}/test05/default.nix | 0 .../operator}/test05/ssh_host_ed25519_key | 0 .../operator}/test05/ssh_host_ed25519_key.pub | 0 .../operator}/test06/default.nix | 0 .../operator}/test06/ssh_host_ed25519_key | 0 .../operator}/test06/ssh_host_ed25519_key.pub | 0 .../operator}/test11/default.nix | 0 .../operator}/test11/ssh_host_ed25519_key | 0 .../operator}/test11/ssh_host_ed25519_key.pub | 0 .../operator}/test12/default.nix | 0 .../operator}/test12/ssh_host_ed25519_key | 0 .../operator}/test12/ssh_host_ed25519_key.pub | 0 .../operator}/test13/default.nix | 0 .../operator}/test13/ssh_host_ed25519_key | 0 .../operator}/test13/ssh_host_ed25519_key.pub | 0 .../operator}/test14/default.nix | 0 .../operator}/test14/ssh_host_ed25519_key | 0 .../operator}/test14/ssh_host_ed25519_key.pub | 0 42 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 machines/README.md rename {infra/machines => machines/dev}/fedi200/default.nix (100%) rename {infra/machines => machines/dev}/fedi201/default.nix (100%) rename {infra/machines => machines/dev}/fedi201/fedipanel.nix (100%) rename {infra/machines => machines/dev}/vm02116/default.nix (100%) rename {infra/machines => machines/dev}/vm02116/forgejo.nix (100%) rename {infra/machines => machines/dev}/vm02187/default.nix (100%) rename {infra/machines => machines/dev}/vm02187/wiki.nix (100%) rename {infra => machines}/machines.md (100%) rename {infra => machines}/machines.md.sh (92%) rename {infra/test-machines => machines/operator}/test01/default.nix (100%) rename {infra/test-machines => machines/operator}/test01/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test01/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test02/default.nix (100%) rename {infra/test-machines => machines/operator}/test02/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test02/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test03/default.nix (100%) rename {infra/test-machines => machines/operator}/test03/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test03/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test04/default.nix (100%) rename {infra/test-machines => machines/operator}/test04/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test04/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test05/default.nix (100%) rename {infra/test-machines => machines/operator}/test05/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test05/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test06/default.nix (100%) rename {infra/test-machines => machines/operator}/test06/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test06/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test11/default.nix (100%) rename {infra/test-machines => machines/operator}/test11/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test11/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test12/default.nix (100%) rename {infra/test-machines => machines/operator}/test12/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test12/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test13/default.nix (100%) rename {infra/test-machines => machines/operator}/test13/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test13/ssh_host_ed25519_key.pub (100%) rename {infra/test-machines => machines/operator}/test14/default.nix (100%) rename {infra/test-machines => machines/operator}/test14/ssh_host_ed25519_key (100%) rename {infra/test-machines => machines/operator}/test14/ssh_host_ed25519_key.pub (100%) diff --git a/infra/README.md b/infra/README.md index 133f6a32..422680fd 100644 --- a/infra/README.md +++ b/infra/README.md @@ -14,7 +14,7 @@ everything will become much cleaner. above 100. For instance, `fedi117`. 2. Add a basic configuration for the machine. These typically go in - `infra/machines//default.nix`. You can look at other `fediXXX` VMs to + `machines/dev//default.nix`. You can look at other `fediXXX` VMs to find inspiration. You probably do not need a `nixos.module` option at this point. @@ -48,7 +48,7 @@ everything will become much cleaner. 7. Regenerate the list of machines: ``` - sh infra/machines.md.sh + sh machines/machines.md.sh ``` Commit it with the machine's configuration, public key, etc. diff --git a/infra/flake-part.nix b/infra/flake-part.nix index af0fe51d..b7bd46f4 100644 --- a/infra/flake-part.nix +++ b/infra/flake-part.nix @@ -28,7 +28,7 @@ let ++ ( if isTestVm then [ - ./test-machines/${vmName} + ../machines/operator/${vmName} { nixos.module.users.users.root.openssh.authorizedKeys.keys = [ # allow our panel vm access to the test machines @@ -38,7 +38,7 @@ let ] else [ - ./machines/${vmName} + ../machines/dev/${vmName} ] ); fediversityVm.name = vmName; @@ -147,8 +147,8 @@ let listSubdirectories = path: attrNames (filterAttrs (_: type: type == "directory") (readDir path)); - machines = listSubdirectories ./machines; - testMachines = listSubdirectories ./test-machines; + machines = listSubdirectories ../machines/dev; + testMachines = listSubdirectories ../machines/operator; in { diff --git a/machines/README.md b/machines/README.md new file mode 100644 index 00000000..d51df126 --- /dev/null +++ b/machines/README.md @@ -0,0 +1,4 @@ +# Machines + +This directory contains the definition of [the VMs](machines.md) that host our +infrastructure. diff --git a/infra/machines/fedi200/default.nix b/machines/dev/fedi200/default.nix similarity index 100% rename from infra/machines/fedi200/default.nix rename to machines/dev/fedi200/default.nix diff --git a/infra/machines/fedi201/default.nix b/machines/dev/fedi201/default.nix similarity index 100% rename from infra/machines/fedi201/default.nix rename to machines/dev/fedi201/default.nix diff --git a/infra/machines/fedi201/fedipanel.nix b/machines/dev/fedi201/fedipanel.nix similarity index 100% rename from infra/machines/fedi201/fedipanel.nix rename to machines/dev/fedi201/fedipanel.nix diff --git a/infra/machines/vm02116/default.nix b/machines/dev/vm02116/default.nix similarity index 100% rename from infra/machines/vm02116/default.nix rename to machines/dev/vm02116/default.nix diff --git a/infra/machines/vm02116/forgejo.nix b/machines/dev/vm02116/forgejo.nix similarity index 100% rename from infra/machines/vm02116/forgejo.nix rename to machines/dev/vm02116/forgejo.nix diff --git a/infra/machines/vm02187/default.nix b/machines/dev/vm02187/default.nix similarity index 100% rename from infra/machines/vm02187/default.nix rename to machines/dev/vm02187/default.nix diff --git a/infra/machines/vm02187/wiki.nix b/machines/dev/vm02187/wiki.nix similarity index 100% rename from infra/machines/vm02187/wiki.nix rename to machines/dev/vm02187/wiki.nix diff --git a/infra/machines.md b/machines/machines.md similarity index 100% rename from infra/machines.md rename to machines/machines.md diff --git a/infra/machines.md.sh b/machines/machines.md.sh similarity index 92% rename from infra/machines.md.sh rename to machines/machines.md.sh index ea1b0208..d523e127 100644 --- a/infra/machines.md.sh +++ b/machines/machines.md.sh @@ -20,7 +20,7 @@ vmOptions=$( cd .. nix eval \ --impure --raw --expr " - builtins.toJSON (builtins.getFlake (builtins.toString ./.)).vmOptions + builtins.toJSON (builtins.getFlake (builtins.toString ../.)).vmOptions " \ --log-format raw --quiet ) diff --git a/infra/test-machines/test01/default.nix b/machines/operator/test01/default.nix similarity index 100% rename from infra/test-machines/test01/default.nix rename to machines/operator/test01/default.nix diff --git a/infra/test-machines/test01/ssh_host_ed25519_key b/machines/operator/test01/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test01/ssh_host_ed25519_key rename to machines/operator/test01/ssh_host_ed25519_key diff --git a/infra/test-machines/test01/ssh_host_ed25519_key.pub b/machines/operator/test01/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test01/ssh_host_ed25519_key.pub rename to machines/operator/test01/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test02/default.nix b/machines/operator/test02/default.nix similarity index 100% rename from infra/test-machines/test02/default.nix rename to machines/operator/test02/default.nix diff --git a/infra/test-machines/test02/ssh_host_ed25519_key b/machines/operator/test02/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test02/ssh_host_ed25519_key rename to machines/operator/test02/ssh_host_ed25519_key diff --git a/infra/test-machines/test02/ssh_host_ed25519_key.pub b/machines/operator/test02/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test02/ssh_host_ed25519_key.pub rename to machines/operator/test02/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test03/default.nix b/machines/operator/test03/default.nix similarity index 100% rename from infra/test-machines/test03/default.nix rename to machines/operator/test03/default.nix diff --git a/infra/test-machines/test03/ssh_host_ed25519_key b/machines/operator/test03/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test03/ssh_host_ed25519_key rename to machines/operator/test03/ssh_host_ed25519_key diff --git a/infra/test-machines/test03/ssh_host_ed25519_key.pub b/machines/operator/test03/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test03/ssh_host_ed25519_key.pub rename to machines/operator/test03/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test04/default.nix b/machines/operator/test04/default.nix similarity index 100% rename from infra/test-machines/test04/default.nix rename to machines/operator/test04/default.nix diff --git a/infra/test-machines/test04/ssh_host_ed25519_key b/machines/operator/test04/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test04/ssh_host_ed25519_key rename to machines/operator/test04/ssh_host_ed25519_key diff --git a/infra/test-machines/test04/ssh_host_ed25519_key.pub b/machines/operator/test04/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test04/ssh_host_ed25519_key.pub rename to machines/operator/test04/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test05/default.nix b/machines/operator/test05/default.nix similarity index 100% rename from infra/test-machines/test05/default.nix rename to machines/operator/test05/default.nix diff --git a/infra/test-machines/test05/ssh_host_ed25519_key b/machines/operator/test05/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test05/ssh_host_ed25519_key rename to machines/operator/test05/ssh_host_ed25519_key diff --git a/infra/test-machines/test05/ssh_host_ed25519_key.pub b/machines/operator/test05/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test05/ssh_host_ed25519_key.pub rename to machines/operator/test05/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test06/default.nix b/machines/operator/test06/default.nix similarity index 100% rename from infra/test-machines/test06/default.nix rename to machines/operator/test06/default.nix diff --git a/infra/test-machines/test06/ssh_host_ed25519_key b/machines/operator/test06/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test06/ssh_host_ed25519_key rename to machines/operator/test06/ssh_host_ed25519_key diff --git a/infra/test-machines/test06/ssh_host_ed25519_key.pub b/machines/operator/test06/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test06/ssh_host_ed25519_key.pub rename to machines/operator/test06/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test11/default.nix b/machines/operator/test11/default.nix similarity index 100% rename from infra/test-machines/test11/default.nix rename to machines/operator/test11/default.nix diff --git a/infra/test-machines/test11/ssh_host_ed25519_key b/machines/operator/test11/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test11/ssh_host_ed25519_key rename to machines/operator/test11/ssh_host_ed25519_key diff --git a/infra/test-machines/test11/ssh_host_ed25519_key.pub b/machines/operator/test11/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test11/ssh_host_ed25519_key.pub rename to machines/operator/test11/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test12/default.nix b/machines/operator/test12/default.nix similarity index 100% rename from infra/test-machines/test12/default.nix rename to machines/operator/test12/default.nix diff --git a/infra/test-machines/test12/ssh_host_ed25519_key b/machines/operator/test12/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test12/ssh_host_ed25519_key rename to machines/operator/test12/ssh_host_ed25519_key diff --git a/infra/test-machines/test12/ssh_host_ed25519_key.pub b/machines/operator/test12/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test12/ssh_host_ed25519_key.pub rename to machines/operator/test12/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test13/default.nix b/machines/operator/test13/default.nix similarity index 100% rename from infra/test-machines/test13/default.nix rename to machines/operator/test13/default.nix diff --git a/infra/test-machines/test13/ssh_host_ed25519_key b/machines/operator/test13/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test13/ssh_host_ed25519_key rename to machines/operator/test13/ssh_host_ed25519_key diff --git a/infra/test-machines/test13/ssh_host_ed25519_key.pub b/machines/operator/test13/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test13/ssh_host_ed25519_key.pub rename to machines/operator/test13/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test14/default.nix b/machines/operator/test14/default.nix similarity index 100% rename from infra/test-machines/test14/default.nix rename to machines/operator/test14/default.nix diff --git a/infra/test-machines/test14/ssh_host_ed25519_key b/machines/operator/test14/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test14/ssh_host_ed25519_key rename to machines/operator/test14/ssh_host_ed25519_key diff --git a/infra/test-machines/test14/ssh_host_ed25519_key.pub b/machines/operator/test14/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test14/ssh_host_ed25519_key.pub rename to machines/operator/test14/ssh_host_ed25519_key.pub -- 2.48.1 From ace56e754e540e6b69628c449ff569f52283ad7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?= Date: Sun, 15 Jun 2025 15:06:23 +0200 Subject: [PATCH 04/37] FediPanel: do not call `nix develop` (#375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yet another piece of #361. Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/375 Reviewed-by: kiara Grouwstra Co-authored-by: Nicolas “Niols” Jeannerod Co-committed-by: Nicolas “Niols” Jeannerod --- panel/default.nix | 7 ++++++- panel/env.nix | 1 - panel/nix/configuration.nix | 22 ++++++++++++++++++---- panel/nix/tests.nix | 1 + panel/src/panel/views.py | 5 ----- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/panel/default.nix b/panel/default.nix index a9c20f84..7b72e360 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -20,8 +20,13 @@ in packages = [ pkgs.npins manage + + # NixOps4 and its dependencies + # FIXME: grab NixOps4 and add it here + pkgs.nix + pkgs.openssh ]; - env = import ./env.nix { inherit lib pkgs; } // { + env = import ./env.nix { } // { NPINS_DIRECTORY = toString ../npins; CREDENTIALS_DIRECTORY = toString ./.credentials; DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; diff --git a/panel/env.nix b/panel/env.nix index 6006016d..c95af586 100644 --- a/panel/env.nix +++ b/panel/env.nix @@ -6,5 +6,4 @@ let in { REPO_DIR = toString ../.; - # explicitly use nix, as e.g. lix does not have configurable-impure-env } diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index 9269cc2a..1996f2df 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -23,7 +23,7 @@ let cfg = config.services.${name}; package = pkgs.callPackage ./package.nix { }; - environment = import ../env.nix { inherit lib pkgs; } // { + environment = import ../env.nix { } // { DATABASE_URL = "sqlite:////var/lib/${name}/db.sqlite3"; USER_SETTINGS_FILE = pkgs.concatText "configuration.py" [ ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) @@ -133,6 +133,17 @@ in type = types.attrsOf types.path; default = { }; }; + nixops4Package = mkOption { + type = types.package; + description = '' + A package providing NixOps4. + + REVIEW: This should not be at the level of the NixOS module, but instead + at the level of the panel's package. Until one finds a way to grab + NixOps4 from the package's npins-based code, we will have to do with + this workaround. + ''; + }; }; config = mkIf cfg.enable { @@ -170,6 +181,8 @@ in }; users.users.${name} = { + # REVIEW[Niols]: change to system user or document why we specifically + # need a normal user. isNormalUser = true; }; @@ -181,10 +194,11 @@ in path = [ python-environment manage-service - # XXX(@fricklerhandwerk): NixOps4 needs a Nix available. + + ## NixOps4 and its dependencies + cfg.nixops4Package pkgs.nix - # TODO(@fricklerhandwerk): Only needed because we invoke NixOps4 via `nix develop`, remove once that's gone. - pkgs.git + pkgs.openssh ]; preStart = '' # Auto-migrate on first run or if the package has changed diff --git a/panel/nix/tests.nix b/panel/nix/tests.nix index 11009213..e76eaed0 100644 --- a/panel/nix/tests.nix +++ b/panel/nix/tests.nix @@ -13,6 +13,7 @@ let secrets = { SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret"; }; + nixops4Package = pkgs.hello; # FIXME: actually pass NixOps4 }; virtualisation = { diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index ad0bbf58..84f25430 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -94,11 +94,6 @@ class DeploymentStatus(ConfigurationForm): "DEPLOYMENT": config.json() } cmd = [ - "nix", - "develop", - "--extra-experimental-features", - "configurable-impure-env", - "--command", "nixops4", "apply", "test", -- 2.48.1 From 3a3a0837936eb4b34c3ed6482b4ba900625ac02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?= Date: Sun, 15 Jun 2025 16:55:19 +0200 Subject: [PATCH 05/37] FediPanel: allow configuring flake and deployment (#376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last part of #361. Builds on top of #375. Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/376 Reviewed-by: kiara Grouwstra Co-authored-by: Nicolas “Niols” Jeannerod Co-committed-by: Nicolas “Niols” Jeannerod --- panel/default.nix | 4 +++- panel/env.nix | 9 --------- panel/nix/configuration.nix | 21 ++++++++++++++++++++- panel/src/panel/settings.py | 7 ++++--- panel/src/panel/views.py | 4 ++-- 5 files changed, 29 insertions(+), 16 deletions(-) delete mode 100644 panel/env.nix diff --git a/panel/default.nix b/panel/default.nix index 7b72e360..c6749611 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -26,7 +26,9 @@ in pkgs.nix pkgs.openssh ]; - env = import ./env.nix { } // { + env = { + DEPLOYMENT_FLAKE = ../.; + DEPLOYMENT_NAME = "test"; NPINS_DIRECTORY = toString ../npins; CREDENTIALS_DIRECTORY = toString ./.credentials; DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; diff --git a/panel/env.nix b/panel/env.nix deleted file mode 100644 index c95af586..00000000 --- a/panel/env.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - ... -}: -let - inherit (builtins) toString; -in -{ - REPO_DIR = toString ../.; -} diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index 1996f2df..e4de7b81 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -23,7 +23,9 @@ let cfg = config.services.${name}; package = pkgs.callPackage ./package.nix { }; - environment = import ../env.nix { } // { + environment = { + DEPLOYMENT_FLAKE = cfg.deployment.flake; + DEPLOYMENT_NAME = cfg.deployment.name; DATABASE_URL = "sqlite:////var/lib/${name}/db.sqlite3"; USER_SETTINGS_FILE = pkgs.concatText "configuration.py" [ ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) @@ -144,6 +146,23 @@ in this workaround. ''; }; + + deployment = { + flake = mkOption { + type = types.path; + default = ../..; + description = '' + The path to the flake containing the deployment. This is used to run the deployment button. + ''; + }; + name = mkOption { + type = types.str; + default = "test"; + description = '' + The name of the deployment within the flake. + ''; + }; + }; }; config = mkIf cfg.enable { diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index 14b1a96f..d613e0ec 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -240,6 +240,7 @@ if user_settings_file is not None: # The correct thing to do here would be using a helper function such as with `get_secret()` that will catch the exception and explain what's wrong and where to put the right values. # Replacing the `USER_SETTINGS_FILE` mechanism following the comment there would probably be a good thing. -# path of the root flake to trigger nixops from, see #94. -# to deploy this should be specified, for dev just use a relative path. -repo_dir = env["REPO_DIR"] +# Path of the root flake to trigger nixops from, see #94, and name of the +# deployment. +deployment_flake = env["DEPLOYMENT_FLAKE"] +deployment_name = env["DEPLOYMENT_NAME"] diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 84f25430..2f603002 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -96,13 +96,13 @@ class DeploymentStatus(ConfigurationForm): cmd = [ "nixops4", "apply", - "test", + settings.deployment_name, "--show-trace", "--no-interactive", ] deployment_result = subprocess.run( cmd, - cwd = settings.repo_dir, + cwd = settings.deployment_flake, env = env, stderr = subprocess.STDOUT, ) -- 2.48.1 From 4801433ae0f6436ff260e60799439318aebc290e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?= Date: Tue, 17 Jun 2025 16:34:29 +0200 Subject: [PATCH 06/37] Get rid of the need for `deployer.pub` (#385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tests still work because we manually write the deployer's public key in `/root/.ssh/authorized_keys` on the target machines. In itself, however, the configuration that we push does not allow the deployer to push anything on the target machines. Context: https://git.fediversity.eu/Fediversity/Fediversity/pulls/361#issuecomment-7857 Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/385 Reviewed-by: kiara Grouwstra Co-authored-by: Nicolas “Niols” Jeannerod Co-committed-by: Nicolas “Niols” Jeannerod --- deployment/check/cli/deployer.pub | 1 - deployment/check/common/nixosTest.nix | 1 - 2 files changed, 2 deletions(-) delete mode 100644 deployment/check/cli/deployer.pub diff --git a/deployment/check/cli/deployer.pub b/deployment/check/cli/deployer.pub deleted file mode 100644 index 2303ffcb..00000000 --- a/deployment/check/cli/deployer.pub +++ /dev/null @@ -1 +0,0 @@ -## This is a placeholder file. It will be overwritten by the test. diff --git a/deployment/check/common/nixosTest.nix b/deployment/check/common/nixosTest.nix index ceec2726..aa91ae2e 100644 --- a/deployment/check/common/nixosTest.nix +++ b/deployment/check/common/nixosTest.nix @@ -119,7 +119,6 @@ in with subtest("Configure the deployer key"): deployer.succeed("""mkdir -p ~/.ssh && ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa""") deployer_key = deployer.succeed("cat ~/.ssh/id_rsa.pub").strip() - deployer.succeed(f"echo '{deployer_key}' > ${config.pathFromRoot}/deployer.pub") ${forConcat config.targetMachines (tm: '' ${tm}.succeed(f"mkdir -p /root/.ssh && echo '{deployer_key}' >> /root/.ssh/authorized_keys") '')} -- 2.48.1 From 939f9d961d684aeac86384bbcc90a0e9a32b4f6e Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Tue, 17 Jun 2025 17:11:52 +0200 Subject: [PATCH 07/37] add data model entity: application (#387) part of #103. Co-authored-by: Valentin Gagarin Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/387 Co-authored-by: Kiara Grouwstra Co-committed-by: Kiara Grouwstra --- .forgejo/workflows/ci.yaml | 6 +++++ README.md | 3 --- default.nix | 17 +++++++++++++ deployment/README.md | 7 ++++++ deployment/data-model-test.nix | 45 ++++++++++++++++++++++++++++++++++ deployment/data-model.nix | 43 ++++++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 deployment/data-model-test.nix create mode 100644 deployment/data-model.nix diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 18925ab8..203eca03 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -15,6 +15,12 @@ jobs: - uses: actions/checkout@v4 - run: nix-build -A tests + check-data-model: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix-shell --run 'nix-unit ./deployment/data-model-test.nix' + check-peertube: runs-on: native steps: diff --git a/README.md b/README.md index 3b19751c..d0a94395 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,3 @@ details as to what they are for. As an overview: - [`services/`](./services) contains our effort to make Fediverse applications work seemlessly together in our specific setting. - -- [`website/`](./website) contains the framework and the content of [the - Fediversity website](https://fediversity.eu/) diff --git a/default.nix b/default.nix index 4c71ec49..7c536a03 100644 --- a/default.nix +++ b/default.nix @@ -41,6 +41,23 @@ in shell = pkgs.mkShellNoCC { inherit (pre-commit-check) shellHook; buildInputs = pre-commit-check.enabledPackages; + packages = + let + test-loop = pkgs.writeShellApplication { + name = "test-loop"; + runtimeInputs = [ + pkgs.watchexec + pkgs.nix-unit + ]; + text = '' + watchexec -w ${builtins.toString ./.} -- nix-unit ${builtins.toString ./deployment/data-model-test.nix} "$@" + ''; + }; + in + [ + pkgs.nix-unit + test-loop + ]; }; tests = { diff --git a/deployment/README.md b/deployment/README.md index f3e24276..784273dd 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -3,6 +3,13 @@ This directory contains work to generate a full Fediversity deployment from a minimal configuration. This is different from [`../services/`](../services) that focuses on one machine, providing a polished and unified interface to different Fediverse services. +## Data model + +The core piece of the project is the [Fediversity data model](./data-model.nix), which describes all entities and their interactions. + +What can be done with it is exemplified in the [evaluation tests](./data-model-test.nix). +Run `test-loop` in the development environment when hacking on the data model or adding tests. + ## Checks There are three levels of deployment checks: `basic`, `cli`, `panel`. diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix new file mode 100644 index 00000000..17c85f77 --- /dev/null +++ b/deployment/data-model-test.nix @@ -0,0 +1,45 @@ +let + inherit (import ../default.nix { }) pkgs; + inherit (pkgs) lib; + eval = + module: + (lib.evalModules { + modules = [ + module + ./data-model.nix + ]; + }).config; +in +{ + test-eval = { + expr = + let + example = eval { + runtime-environments.bar.nixos = { + module = + { ... }: + { + system.stateVersion = "25.05"; + }; + }; + applications.foo = { + module = + { pkgs, ... }: + { + environment.systemPackages = [ + pkgs.hello + ]; + }; + }; + }; + in + { + has-runtime = lib.isAttrs example.runtime-environments.bar.nixos.module; + has-application = lib.isAttrs example.applications.foo.module; + }; + expected = { + has-runtime = true; + has-application = true; + }; + }; +} diff --git a/deployment/data-model.nix b/deployment/data-model.nix new file mode 100644 index 00000000..af867d55 --- /dev/null +++ b/deployment/data-model.nix @@ -0,0 +1,43 @@ +{ + lib, + ... +}: +let + inherit (lib) types mkOption; +in +with types; +{ + options = { + runtime-environments = mkOption { + description = "Collection of runtime environments into which applications can be deployed"; + type = attrsOf (attrTag { + nixos = mkOption { + description = "A single NixOS machine"; + type = submodule { + options = { + module = mkOption { + description = "The NixOS module describing the base configuration for that machine"; + type = deferredModule; + }; + }; + }; + }; + }); + }; + applications = mkOption { + description = "Collection of Fediversity applications"; + type = attrsOf (submoduleWith { + modules = [ + { + options = { + module = mkOption { + description = "The NixOS module for that application, for configuring that application"; + type = deferredModule; + }; + }; + } + ]; + }); + }; + }; +} -- 2.48.1 From bd1cfd7a7cc5699cf2e70f16540d204edfc01599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?= Date: Wed, 18 Jun 2025 12:37:47 +0200 Subject: [PATCH 08/37] Introduce test for deploying all services via FediPanel (#361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #277 Same as #329 but where we run the FediPanel and interact with it via a browser instead of running NixOps4 directly. Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/361 Reviewed-by: kiara Grouwstra Reviewed-by: Valentin Gagarin Co-authored-by: Nicolas “Niols” Jeannerod Co-committed-by: Nicolas “Niols” Jeannerod --- .forgejo/workflows/ci.yaml | 6 + deployment/README.md | 6 +- deployment/check/panel/flake-part.nix | 91 +++++++ deployment/check/panel/nixosTest.nix | 362 ++++++++++++++++++++++++++ deployment/flake-part.nix | 1 + panel/nix/configuration.nix | 4 +- panel/src/panel/settings.py | 1 + 7 files changed, 466 insertions(+), 5 deletions(-) create mode 100644 deployment/check/panel/flake-part.nix create mode 100644 deployment/check/panel/nixosTest.nix diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 203eca03..f7f0a438 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -44,3 +44,9 @@ jobs: steps: - uses: actions/checkout@v4 - run: nix build .#checks.x86_64-linux.deployment-cli -L + + check-deployment-panel: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix build .#checks.x86_64-linux.deployment-panel -L diff --git a/deployment/README.md b/deployment/README.md index 784273dd..f8eeabdc 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -116,8 +116,8 @@ flowchart LR target_machines -->|get certs| acme ``` -### [WIP] Service deployment check from the panel +### Service deployment check from the FediPanel -This is a full deployment check running the panel on the deployer machine, deploying some services through the panel and checking that they are indeed on the target machines, then cleans them up and checks whether that works, too. +This is a full deployment check running the [FediPanel](../panel) on the deployer machine, deploying some services through it and checking that they are indeed on the target machines, then cleans them up and checks whether that works, too. -It builds upon the basic and CLI deployment checks. +It builds upon the basic and CLI deployment checks, the only difference being that `deployer` runs NixOps4 only indirectly via the panel, and the `client` node is the one that triggers the deployment via a browser, the way a human would. diff --git a/deployment/check/panel/flake-part.nix b/deployment/check/panel/flake-part.nix new file mode 100644 index 00000000..24a3d695 --- /dev/null +++ b/deployment/check/panel/flake-part.nix @@ -0,0 +1,91 @@ +{ + self, + inputs, + lib, + ... +}: + +let + inherit (builtins) + fromJSON + listToAttrs + ; + + targetMachines = [ + "garage" + "mastodon" + "peertube" + "pixelfed" + ]; + pathToRoot = /. + (builtins.unsafeDiscardStringContext self); + pathFromRoot = ./.; + enableAcme = true; + +in +{ + perSystem = + { pkgs, ... }: + { + checks.deployment-panel = pkgs.testers.runNixOSTest { + imports = [ + ../common/nixosTest.nix + ./nixosTest.nix + ]; + _module.args.inputs = inputs; + inherit + targetMachines + pathToRoot + pathFromRoot + enableAcme + ; + }; + }; + + nixops4Deployments = + let + makeTargetResource = nodeName: { + imports = [ ../common/targetResource.nix ]; + _module.args.inputs = inputs; + inherit + nodeName + pathToRoot + pathFromRoot + enableAcme + ; + }; + + ## The deployment function - what we are here to test! + ## + ## TODO: Modularise `deployment/default.nix` to get rid of the nested + ## function calls. + makeTestDeployment = + args: + (import ../..) + { + inherit lib; + inherit (inputs) nixops4 nixops4-nixos; + fediversity = import ../../../services/fediversity; + } + (listToAttrs ( + map (nodeName: { + name = "${nodeName}ConfigurationResource"; + value = makeTargetResource nodeName; + }) targetMachines + )) + args; + + in + { + check-deployment-panel = makeTestDeployment ( + fromJSON ( + let + env = builtins.getEnv "DEPLOYMENT"; + in + if env == "" then + throw "The DEPLOYMENT environment needs to be set. You do not want to use this deployment unless in the `deployment-panel` NixOS test." + else + env + ) + ); + }; +} diff --git a/deployment/check/panel/nixosTest.nix b/deployment/check/panel/nixosTest.nix new file mode 100644 index 00000000..39bd7930 --- /dev/null +++ b/deployment/check/panel/nixosTest.nix @@ -0,0 +1,362 @@ +{ + inputs, + lib, + hostPkgs, + config, + ... +}: + +let + inherit (lib) + getExe + ; + + ## Some places need a dummy file that will in fact never be used. We create + ## it here. + dummyFile = hostPkgs.writeText "dummy" "dummy"; + panelPort = 8000; + + panelUser = "test"; + panelEmail = "test@test.com"; + panelPassword = "ouiprdaaa43"; # panel's manager complains if too close to username or email + + fediUser = "test"; + fediEmail = "test@test.com"; + fediPassword = "testtest"; + fediName = "Testy McTestface"; + + toPythonBool = b: if b then "True" else "False"; + + interactWithPanel = + { + baseUri, + enableMastodon, + enablePeertube, + enablePixelfed, + }: + hostPkgs.writers.writePython3Bin "interact-with-panel" + { + libraries = with hostPkgs.python3Packages; [ selenium ]; + flakeIgnore = [ + "E302" # expected 2 blank lines, found 0 + "E303" # too many blank lines + "E305" # expected 2 blank lines after end of function or class + "E501" # line too long (> 79 characters) + "E731" # do not assign lambda expression, use a def + ]; + } + '' + from selenium import webdriver + from selenium.webdriver.common.by import By + from selenium.webdriver.firefox.options import Options + from selenium.webdriver.support.ui import WebDriverWait + + print("Create and configure driver...") + options = Options() + options.add_argument("--headless") + options.binary_location = "${getExe hostPkgs.firefox-unwrapped}" + service = webdriver.FirefoxService(executable_path="${getExe hostPkgs.geckodriver}") + driver = webdriver.Firefox(options=options, service=service) + driver.set_window_size(1280, 960) + driver.implicitly_wait(360) + driver.command_executor.set_timeout(3600) + + print("Open login page...") + driver.get("${baseUri}/login/") + print("Enter username...") + driver.find_element(By.XPATH, "//input[@name = 'username']").send_keys("${panelUser}") + print("Enter password...") + driver.find_element(By.XPATH, "//input[@name = 'password']").send_keys("${panelPassword}") + print("Click “Login” button...") + driver.find_element(By.XPATH, "//button[normalize-space() = 'Login']").click() + + print("Open configuration page...") + driver.get("${baseUri}/configuration/") + + # Helpers to actually set and not add or switch input values. + def input_set(elt, keys): + elt.clear() + elt.send_keys(keys) + def checkbox_set(elt, new_value): + if new_value != elt.is_selected(): + elt.click() + + print("Enable Fediversity...") + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'enable']"), True) + + print("Fill in initialUser info...") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.username']"), "${fediUser}") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.password']"), "${fediPassword}") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.email']"), "${fediEmail}") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.displayName']"), "${fediName}") + + print("Enable services...") + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'mastodon.enable']"), ${toPythonBool enableMastodon}) + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'peertube.enable']"), ${toPythonBool enablePeertube}) + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'pixelfed.enable']"), ${toPythonBool enablePixelfed}) + + print("Start deployment...") + driver.find_element(By.XPATH, "//button[@id = 'deploy-button']").click() + + print("Wait for deployment status to show up...") + get_deployment_result = lambda d: d.find_element(By.XPATH, "//div[@id = 'deployment-result']//p") + WebDriverWait(driver, timeout=3660, poll_frequency=10).until(get_deployment_result) + deployment_result = get_deployment_result(driver).get_attribute('innerHTML') + + print("Quit...") + driver.quit() + + match deployment_result: + case 'Deployment Succeeded': + print("Deployment has succeeded; exiting normally") + exit(0) + case 'Deployment Failed': + print("Deployment has failed; exiting with return code `1`") + exit(1) + case _: + print(f"Unexpected deployment result: {deployment_result}; exiting with return code `2`") + exit(2) + ''; + +in + +{ + name = "deployment-panel"; + + ## The panel's module sets `nixpkgs.overlays` which clashes with + ## `pkgsReadOnly`. We disable it here. + node.pkgsReadOnly = false; + + nodes.deployer = + { pkgs, ... }: + { + imports = [ + (import ../../../panel { }).module + ]; + + ## FIXME: This should be in the common stuff. + security.acme = { + acceptTerms = true; + defaults.email = "test@test.com"; + defaults.server = "https://acme.test/dir"; + }; + security.pki.certificateFiles = [ + (import "${inputs.nixpkgs}/nixos/tests/common/acme/server/snakeoil-certs.nix").ca.cert + ]; + networking.extraHosts = "${config.acmeNodeIP} acme.test"; + + services.panel = { + enable = true; + production = true; + domain = "deployer"; + secrets = { + SECRET_KEY = dummyFile; + }; + port = panelPort; + nixops4Package = inputs.nixops4.packages.${pkgs.system}.default; + + deployment = { + flake = "/run/fedipanel/flake"; + name = "check-deployment-panel"; + }; + }; + + environment.systemPackages = [ pkgs.expect ]; + + ## FIXME: The following dependencies are necessary but I do not + ## understand why they are not covered by the fake node. + system.extraDependencies = with pkgs; [ + peertube + peertube.inputDerivation + gixy # a configuration checker for nginx + gixy.inputDerivation + ]; + + system.extraDependenciesFromModule = { + imports = [ ../../../services/fediversity ]; + fediversity = { + domain = "fediversity.net"; # would write `dummy` but that would not type + garage.enable = true; + mastodon = { + enable = true; + s3AccessKeyFile = dummyFile; + s3SecretKeyFile = dummyFile; + }; + peertube = { + enable = true; + secretsFile = dummyFile; + s3AccessKeyFile = dummyFile; + s3SecretKeyFile = dummyFile; + }; + pixelfed = { + enable = true; + s3AccessKeyFile = dummyFile; + s3SecretKeyFile = dummyFile; + }; + temp.cores = 1; + temp.initialUser = { + username = "dummy"; + displayName = "dummy"; + email = "dummy"; + passwordFile = dummyFile; + }; + }; + }; + }; + + nodes.client = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + httpie + dnsutils # for `dig` + openssl + cacert + wget + python3 + python3Packages.selenium + firefox-unwrapped + geckodriver + ]; + + security.pki.certificateFiles = [ + config.nodes.acme.test-support.acme.caCert + ]; + networking.extraHosts = "${config.acmeNodeIP} acme.test"; + }; + + ## NOTE: The target machines may need more RAM than the default to handle + ## being deployed to, otherwise we get something like: + ## + ## pixelfed # [ 616.785499 ] sshd-session[1167]: Conection closed by 2001:db8:1::2 port 45004 + ## deployer # error: writing to file: No space left on device + ## pixelfed # [ 616.788538 ] sshd-session[1151]: pam_unix(sshd:session): session closed for user port + ## pixelfed # [ 616.793929 ] systemd-logind[719]: Session 4 logged out. Waiting for processes to exit. + ## deployer # Error: Could not create resource + ## + ## These values have been trimmed down to the gigabyte. + nodes.mastodon.virtualisation.memorySize = 4 * 1024; + nodes.pixelfed.virtualisation.memorySize = 4 * 1024; + nodes.peertube.virtualisation.memorySize = 5 * 1024; + + ## FIXME: The test of presence of the services are very simple: we only + ## check that there is a systemd service of the expected name on the + ## machine. This proves at least that NixOps4 did something, and we cannot + ## really do more for now because the services aren't actually working + ## properly, in particular because of DNS issues. We should fix the services + ## and check that they are working properly. + + extraTestScript = '' + ## TODO: We want a nicer way to control where the FediPanel consumes its + ## flake, which can default to the store but could also be somewhere else if + ## someone wanted to change the code of the flake. + ## + with subtest("Give the panel access to the flake"): + deployer.succeed("mkdir /run/fedipanel /run/fedipanel/flake >&2") + deployer.succeed("cp -R . /run/fedipanel/flake >&2") + deployer.succeed("chown -R panel:panel /run/fedipanel >&2") + + ## TODO: I want a programmatic way to provide an SSH key to the panel (and + ## therefore NixOps4). This should happen either in the Python code, but + ## maybe it is fair that that one picks up on the user's key? It could + ## also be in the Nix packaging. + ## + with subtest("Set up the panel's SSH keys"): + deployer.succeed("mkdir /home/panel/.ssh >&2") + deployer.succeed("cp -R /root/.ssh/* /home/panel/.ssh >&2") + deployer.succeed("chown -R panel:panel /home/panel/.ssh >&2") + deployer.succeed("chmod 600 /home/panel/.ssh/* >&2") + + ## TODO: This is a hack to accept the root CA used by Pebble on the client + ## machine. Pebble randomizes everything, so the only way to get it is to + ## call the /roots/0 endpoint at runtime, leaving not much margin for a nice + ## Nixy way of adding the certificate. There is no way around it as this is + ## by design in Pebble, showing in fact that Pebble was not the appropriate + ## tool for our use and that nixpkgs does not in fact provide an easy way to + ## generate _usable_ certificates in NixOS tests. I suggest we merge this, + ## and track the task to set it up in a cleaner way. I would tackle this in + ## a subsequent PR, and hopefully even contribute this BetterWay(tm) to + ## nixpkgs. — Niols + ## + with subtest("Set up ACME root CA on client"): + client.succeed(""" + cd /etc/ssl/certs + curl -o pebble-root-ca.pem https://acme.test:15000/roots/0 + curl -o pebble-intermediate-ca.pem https://acme.test:15000/intermediates/0 + { cat ca-bundle.crt + cat pebble-root-ca.pem + cat pebble-intermediate-ca.pem + } > new-ca-bundle.crt + rm ca-bundle.crt ca-certificates.crt + mv new-ca-bundle.crt ca-bundle.crt + ln -s ca-bundle.crt ca-certificates.crt + """) + + ## TODO: I would hope for a more declarative way to add users. This should + ## be handled by the Nix packaging of the FediPanel. — Niols + ## + with subtest("Create panel user"): + deployer.succeed(""" + expect -c ' + spawn manage createsuperuser --username ${panelUser} --email ${panelEmail} + expect "Password: "; send "${panelPassword}\\n"; + expect "Password (again): "; send "${panelPassword}\\n" + interact + ' >&2 + """) + + with subtest("Check the status of the services - there should be none"): + garage.fail("systemctl status garage.service") + mastodon.fail("systemctl status mastodon-web.service") + peertube.fail("systemctl status peertube.service") + pixelfed.fail("systemctl status phpfpm-pixelfed.service") + + with subtest("Run deployment with no services enabled"): + client.succeed("${ + interactWithPanel { + baseUri = "https://deployer"; + enableMastodon = false; + enablePeertube = false; + enablePixelfed = false; + } + }/bin/interact-with-panel >&2") + + with subtest("Check the status of the services - there should still be none"): + garage.fail("systemctl status garage.service") + mastodon.fail("systemctl status mastodon-web.service") + peertube.fail("systemctl status peertube.service") + pixelfed.fail("systemctl status phpfpm-pixelfed.service") + + with subtest("Run deployment with Mastodon and Pixelfed enabled"): + client.succeed("${ + interactWithPanel { + baseUri = "https://deployer"; + enableMastodon = true; + enablePeertube = false; + enablePixelfed = true; + } + }/bin/interact-with-panel >&2") + + with subtest("Check the status of the services - expecting Garage, Mastodon and Pixelfed"): + garage.succeed("systemctl status garage.service") + mastodon.succeed("systemctl status mastodon-web.service") + peertube.fail("systemctl status peertube.service") + pixelfed.succeed("systemctl status phpfpm-pixelfed.service") + + with subtest("Run deployment with only Peertube enabled"): + client.succeed("${ + interactWithPanel { + baseUri = "https://deployer"; + enableMastodon = false; + enablePeertube = true; + enablePixelfed = false; + } + }/bin/interact-with-panel >&2") + + with subtest("Check the status of the services - expecting Garage and Peertube"): + garage.succeed("systemctl status garage.service") + mastodon.fail("systemctl status mastodon-web.service") + peertube.succeed("systemctl status peertube.service") + pixelfed.fail("systemctl status phpfpm-pixelfed.service") + ''; +} diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix index 5e822688..4f31f2eb 100644 --- a/deployment/flake-part.nix +++ b/deployment/flake-part.nix @@ -2,5 +2,6 @@ imports = [ ./check/basic/flake-part.nix ./check/cli/flake-part.nix + ./check/panel/flake-part.nix ]; } diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index e4de7b81..ccc6506a 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -140,7 +140,7 @@ in description = '' A package providing NixOps4. - REVIEW: This should not be at the level of the NixOS module, but instead + TODO: This should not be at the level of the NixOS module, but instead at the level of the panel's package. Until one finds a way to grab NixOps4 from the package's npins-based code, we will have to do with this workaround. @@ -200,7 +200,7 @@ in }; users.users.${name} = { - # REVIEW[Niols]: change to system user or document why we specifically + # TODO[Niols]: change to system user or document why we specifically # need a normal user. isNormalUser = true; }; diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index d613e0ec..5a555495 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -42,6 +42,7 @@ def get_secret(name: str, encoding: str = "utf-8") -> str: return secret # SECURITY WARNING: keep the secret key used in production secret! +# This is used nowhere but is required by Django. SECRET_KEY = get_secret("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -- 2.48.1 From d67f533948b2d9ae72b7015443332e3ef8700682 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Jun 2025 08:26:20 +0200 Subject: [PATCH 09/37] fix running `nixops4 apply test` (#391) Closes #390 Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/391 Reviewed-by: kiara Grouwstra Co-authored-by: Valentin Gagarin Co-committed-by: Valentin Gagarin --- infra/common/resource.nix | 5 +++-- infra/flake-part.nix | 3 +++ npins/sources.json | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/infra/common/resource.nix b/infra/common/resource.nix index 7e86467c..ebaea6c6 100644 --- a/infra/common/resource.nix +++ b/infra/common/resource.nix @@ -1,4 +1,5 @@ { + inputs, lib, config, ... @@ -9,7 +10,7 @@ let inherit (lib.attrsets) concatMapAttrs optionalAttrs; inherit (lib.strings) removeSuffix; sources = import ../../npins; - inherit (sources) nixpkgs agenix disko; + inherit (sources) agenix disko; secretsPrefix = ../../secrets; secrets = import (secretsPrefix + "/secrets.nix"); @@ -26,7 +27,7 @@ in hostPublicKey = config.fediversityVm.hostPublicKey; }; - inherit nixpkgs; + inherit (inputs) nixpkgs; ## The configuration of the machine. We strive to keep in this file only the ## options that really need to be injected from the resource. Everything else diff --git a/infra/flake-part.nix b/infra/flake-part.nix index b7bd46f4..068bf6a6 100644 --- a/infra/flake-part.nix +++ b/infra/flake-part.nix @@ -21,6 +21,9 @@ let makeResourceModule = { vmName, isTestVm }: { + # TODO(@fricklerhandwerk): this is terrible but IMO we should just ditch flake-parts and have our own data model for how the project is organised internally + _module.args = { inherit inputs; }; + imports = [ ./common/resource.nix diff --git a/npins/sources.json b/npins/sources.json index 657e8a3a..a96ffcb2 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -25,6 +25,22 @@ "url": null, "hash": "1w2gsy6qwxa5abkv8clb435237iifndcxq0s79wihqw11a5yb938" }, + "disko": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "nix-community", + "repo": "disko" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "v1.12.0", + "revision": "7121f74b976481bc36877abaf52adab2a178fcbe", + "url": "https://api.github.com/repos/nix-community/disko/tarball/v1.12.0", + "hash": "0wbx518d2x54yn4xh98cgm65wvj0gpy6nia6ra7ns4j63hx14fkq" + }, "flake-inputs": { "type": "GitRelease", "repository": { -- 2.48.1 From 611c961dcfa462f8261fafe1f67d86a218ed7b04 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Thu, 19 Jun 2025 18:11:08 +0200 Subject: [PATCH 10/37] separate test declarations from invocations (#396) see https://git.fediversity.eu/Fediversity/Fediversity/pulls/395#issuecomment-8024 Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/396 Reviewed-by: Valentin Gagarin Co-authored-by: Kiara Grouwstra Co-committed-by: Kiara Grouwstra --- services/default.nix | 6 +++--- services/tests/mastodon.nix | 2 +- services/tests/peertube.nix | 2 +- services/tests/pixelfed-garage.nix | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/default.nix b/services/default.nix index a1c868ad..3117b861 100644 --- a/services/default.nix +++ b/services/default.nix @@ -6,8 +6,8 @@ }: { tests = { - mastodon = import ./tests/mastodon.nix { inherit pkgs; }; - pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit pkgs; }; - peertube = import ./tests/peertube.nix { inherit pkgs; }; + mastodon = pkgs.nixosTest ./tests/mastodon.nix; + pixelfed-garage = pkgs.nixosTest ./tests/pixelfed-garage.nix; + peertube = pkgs.nixosTest ./tests/peertube.nix; }; } diff --git a/services/tests/mastodon.nix b/services/tests/mastodon.nix index 244f0304..f5497520 100644 --- a/services/tests/mastodon.nix +++ b/services/tests/mastodon.nix @@ -42,7 +42,7 @@ let ''; in -pkgs.nixosTest { +{ name = "mastodon"; nodes = { diff --git a/services/tests/peertube.nix b/services/tests/peertube.nix index 27d79589..475c3f54 100644 --- a/services/tests/peertube.nix +++ b/services/tests/peertube.nix @@ -161,7 +161,7 @@ let ''; in -pkgs.nixosTest { +{ name = "peertube"; nodes = { diff --git a/services/tests/pixelfed-garage.nix b/services/tests/pixelfed-garage.nix index 13ad1ef7..66116774 100644 --- a/services/tests/pixelfed-garage.nix +++ b/services/tests/pixelfed-garage.nix @@ -114,7 +114,7 @@ let ${seleniumQuit}''; in -pkgs.nixosTest { +{ name = "test-pixelfed-garage"; nodes = { -- 2.48.1 From 486b3168853ec0b474920cdb0f9bc80f8462ba02 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Fri, 20 Jun 2025 09:41:38 +0200 Subject: [PATCH 11/37] run updater natively (#394) see https://git.fediversity.eu/Fediversity/Fediversity/issues/65#issuecomment-7668. closes #65. Reviewed-on: https://git.fediversity.eu/Fediversity/Fediversity/pulls/394 Co-authored-by: Kiara Grouwstra Co-committed-by: Kiara Grouwstra --- .forgejo/workflows/update.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/update.yaml b/.forgejo/workflows/update.yaml index d76c9622..ac9a17f4 100644 --- a/.forgejo/workflows/update.yaml +++ b/.forgejo/workflows/update.yaml @@ -7,15 +7,16 @@ on: jobs: lockfile: - runs-on: ubuntu-latest + runs-on: native steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Nix - uses: cachix/install-nix-action@v31 - name: Install npins - run: nix profile install 'nixpkgs#npins' - - name: Update npins sources - uses: getchoo/update-npins@v0 + run: nix-shell --run "npins update" + - name: Create PR + uses: peter-evans/create-pull-request@v7 with: token: "${{ secrets.DEPLOY_KEY }}" + branch: npins-update + commit-message: "npins: update sources" + title: "npins: update sources" -- 2.48.1 From 8b2ee21dbe0099ac3cfef9f27b6408d6e484ab12 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 23 Jun 2025 09:06:52 +0200 Subject: [PATCH 12/37] data model: add run-time configuration --- deployment/data-model-test.nix | 21 ++++++---- deployment/data-model.nix | 75 ++++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 17c85f77..cc9b83fa 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -11,17 +11,18 @@ let }).config; in { + _class = "nix-unit"; + test-eval = { expr = let example = eval { - runtime-environments.bar.nixos = { - module = - { ... }: - { - system.stateVersion = "25.05"; - }; - }; + runtime-configurations.nixos = + { ... }: + { + system.stateVersion = "25.05"; + }; + runtime-environments.bar.nixos = { }; applications.foo = { module = { pkgs, ... }: @@ -34,11 +35,13 @@ in }; in { - has-runtime = lib.isAttrs example.runtime-environments.bar.nixos.module; + has-runtime-configuration = lib.isAttrs example.runtime-configurations.nixos; + has-runtime-environment = lib.isAttrs example.runtime-environments.bar.nixos.module; has-application = lib.isAttrs example.applications.foo.module; }; expected = { - has-runtime = true; + has-runtime-configuration = true; + has-runtime-environment = true; has-application = true; }; }; diff --git a/deployment/data-model.nix b/deployment/data-model.nix index af867d55..81738e45 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -1,43 +1,68 @@ { lib, + config, ... }: let - inherit (lib) types mkOption; -in -with types; -{ - options = { - runtime-environments = mkOption { - description = "Collection of runtime environments into which applications can be deployed"; - type = attrsOf (attrTag { - nixos = mkOption { - description = "A single NixOS machine"; - type = submodule { - options = { - module = mkOption { - description = "The NixOS module describing the base configuration for that machine"; - type = deferredModule; - }; - }; - }; - }; - }); + inherit (lib) attrNames mkOption genAttrs; + inherit (lib.types) + attrsOf + attrTag + deferredModule + submoduleWith + ; + runtime-configuration = mkOption { + description = "The NixOS module of a run-time environment"; + type = deferredModule; + default = { + _class = "nixos"; }; - applications = mkOption { - description = "Collection of Fediversity applications"; - type = attrsOf (submoduleWith { + }; + runtime-environment = attrTag { + nixos = mkOption { + type = submoduleWith { modules = [ { options = { module = mkOption { - description = "The NixOS module for that application, for configuring that application"; + description = "The NixOS module of the run-time environment"; type = deferredModule; + default = config.runtime-configurations.nixos; + readOnly = true; }; }; } ]; - }); + }; + }; + }; + application = submoduleWith { + description = "A Fediversity application"; + modules = [ + { + options = { + module = mkOption { + description = "The NixOS module for that application, for configuring that application"; + type = deferredModule; + }; + }; + } + ]; + }; +in +{ + options = { + runtime-configurations = mkOption { + description = "Collection of runtime environments into which applications can be deployed"; + type = attrTag (genAttrs (attrNames runtime-environment.nestedTypes) (_: runtime-configuration)); + }; + runtime-environments = mkOption { + description = "Collection of runtime environments into which applications can be deployed"; + type = attrsOf runtime-environment; + }; + applications = mkOption { + description = "Collection of Fediversity applications"; + type = attrsOf application; }; }; } -- 2.48.1 From c1f3aa6aed0d191b512d5030000064dc97c576e5 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 23 Jun 2025 09:34:59 +0200 Subject: [PATCH 13/37] have run-time environments use their corresponding run-time configurations --- deployment/data-model.nix | 43 +++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 81738e45..773c0f91 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -18,24 +18,31 @@ let _class = "nixos"; }; }; - runtime-environment = attrTag { - nixos = mkOption { - type = submoduleWith { - modules = [ - { - options = { - module = mkOption { - description = "The NixOS module of the run-time environment"; - type = deferredModule; - default = config.runtime-configurations.nixos; - readOnly = true; - }; - }; - } - ]; - }; - }; - }; + runtime-environment = attrTag ( + mapAttrs + ( + name: options: + mkOption { + type = submoduleWith { + modules = [ + { + options = options // { + module = mkOption { + description = "The NixOS module of the run-time environment"; + type = deferredModule; + default = config.runtime-configurations.${name}; + readOnly = true; + }; + }; + } + ]; + }; + } + ) + { + nixos = { }; + } + ); application = submoduleWith { description = "A Fediversity application"; modules = [ -- 2.48.1 From fefcd93bc177b2b7f324645bbc37f37ee4d8a306 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 23 Jun 2025 11:25:18 +0200 Subject: [PATCH 14/37] grant run-time environments their own modules with their own description --- deployment/data-model.nix | 47 +++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 773c0f91..b6e11a17 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -4,12 +4,19 @@ ... }: let - inherit (lib) attrNames mkOption genAttrs; + inherit (lib) + attrNames + mapAttrs + mkOption + genAttrs + ; inherit (lib.types) attrsOf attrTag deferredModule + mergeTypes submoduleWith + submodule ; runtime-configuration = mkOption { description = "The NixOS module of a run-time environment"; @@ -21,26 +28,28 @@ let runtime-environment = attrTag ( mapAttrs ( - name: options: - mkOption { - type = submoduleWith { - modules = [ - { - options = options // { - module = mkOption { - description = "The NixOS module of the run-time environment"; - type = deferredModule; - default = config.runtime-configurations.${name}; - readOnly = true; - }; - }; - } - ]; - }; - } + name: + option@{ type, ... }: + mkOption ( + option + // { + type = mergeTypes type (submodule { + options.module = mkOption { + description = "The NixOS module of the run-time environment"; + type = deferredModule; + default = config.runtime-configurations.${name}; + readOnly = true; + }; + }); + } + ) ) { - nixos = { }; + nixos = { + description = "A NixOS instance to deploy to."; + type = submodule { + }; + }; } ); application = submoduleWith { -- 2.48.1 From f51462afc96d5e2c1ae905852af5f0e472d1626b Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 23 Jun 2025 16:10:17 +0200 Subject: [PATCH 15/37] data model: runtime environment allows declaring options so instantiations may configure required settings --- deployment/data-model-test.nix | 16 +++++++++--- deployment/data-model.nix | 46 ++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index cc9b83fa..9163eafb 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -17,12 +17,20 @@ in expr = let example = eval { - runtime-configurations.nixos = + runtime-configurations.single-ssh-host = { ... }: { system.stateVersion = "25.05"; }; - runtime-environments.bar.nixos = { }; + runtime-environments.bar = { + single-ssh-host = { + ssh = { + host = "localhost"; + username = "root"; + authentication.password = ""; + }; + }; + }; applications.foo = { module = { pkgs, ... }: @@ -35,8 +43,8 @@ in }; in { - has-runtime-configuration = lib.isAttrs example.runtime-configurations.nixos; - has-runtime-environment = lib.isAttrs example.runtime-environments.bar.nixos.module; + has-runtime-configuration = lib.isAttrs example.runtime-configurations.single-ssh-host; + has-runtime-environment = lib.isAttrs example.runtime-environments.bar.single-ssh-host.module; has-application = lib.isAttrs example.applications.foo.module; }; expected = { diff --git a/deployment/data-model.nix b/deployment/data-model.nix index b6e11a17..e0ed15c2 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -15,8 +15,10 @@ let attrTag deferredModule mergeTypes + nullOr submoduleWith submodule + str ; runtime-configuration = mkOption { description = "The NixOS module of a run-time environment"; @@ -45,9 +47,49 @@ let ) ) { - nixos = { - description = "A NixOS instance to deploy to."; + vm = { + description = "A VM to deploy to."; type = submodule { + options = { + }; + }; + }; + single-ssh-host = { + description = "A single host to deploy to by SSH."; + type = submodule { + options = { + ssh = mkOption { + description = "SSH connection info"; + type = submodule { + options = { + host = mkOption { + description = "the host to access by SSH"; + type = str; + }; + username = mkOption { + description = "the SSH user to use"; + type = nullOr str; + default = null; + }; + authentication = mkOption { + description = "authentication method"; + type = attrsOf (attrTag { + private-key = mkOption { + description = "path to the user's SSH private key"; + type = str; + example = "/root/.ssh/id_ed25519"; + }; + password = mkOption { + description = "SSH password"; + # TODO: mark as sensitive + type = str; + }; + }); + }; + }; + }; + }; + }; }; }; } -- 2.48.1 From 6c2022d06479443a44d863a9e5695522a833e357 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 23 Jun 2025 16:33:36 +0200 Subject: [PATCH 16/37] data model: deployment --- deployment/data-model-test.nix | 55 ++++++++++++++++++++-------------- deployment/data-model.nix | 24 +++++++++++++++ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 9163eafb..55a0e1d8 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -16,41 +16,50 @@ in test-eval = { expr = let - example = eval { - runtime-configurations.single-ssh-host = - { ... }: - { - system.stateVersion = "25.05"; - }; - runtime-environments.bar = { - single-ssh-host = { - ssh = { - host = "localhost"; - username = "root"; - authentication.password = ""; - }; - }; - }; - applications.foo = { - module = - { pkgs, ... }: + example = eval ( + { config, ... }: + { + runtime-configurations.single-ssh-host = + { ... }: { - environment.systemPackages = [ - pkgs.hello - ]; + system.stateVersion = "25.05"; }; - }; - }; + runtime-environments.bar = { + single-ssh-host = { + ssh = { + host = "localhost"; + username = "root"; + authentication.password = ""; + }; + }; + }; + applications.foo = { + module = + { pkgs, ... }: + { + environment.systemPackages = [ + pkgs.hello + ]; + }; + }; + deployments.baz = { + module = { }; + runtime-environment = config.runtime-environments.bar; + }; + } + ); in { has-runtime-configuration = lib.isAttrs example.runtime-configurations.single-ssh-host; has-runtime-environment = lib.isAttrs example.runtime-environments.bar.single-ssh-host.module; has-application = lib.isAttrs example.applications.foo.module; + has-deployment = lib.isAttrs example.deployments.baz.module; }; expected = { has-runtime-configuration = true; has-runtime-environment = true; has-application = true; + has-deployment = true; }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index e0ed15c2..2a90fc11 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -107,6 +107,26 @@ let } ]; }; + deployment = submoduleWith { + description = "A deployment of a configuration of applications to a run-time environment"; + modules = [ + { + options = { + # the `applications` option consists of configuration for the above applications + module = mkOption { + description = '' + Configuration to be deployed + ''; + type = deferredModule; + }; + runtime-environment = mkOption { + description = "The run-time environment to deploy to"; + type = runtime-environment; + }; + }; + } + ]; + }; in { options = { @@ -122,5 +142,9 @@ in description = "Collection of Fediversity applications"; type = attrsOf application; }; + deployments = mkOption { + description = "Deployment of a configuration of applications to a run-time environment"; + type = attrsOf deployment; + }; }; } -- 2.48.1 From 34529a7de434b5a1d79049d17ada3d80c8420e14 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 23 Jun 2025 19:22:47 +0200 Subject: [PATCH 17/37] data model: migration --- deployment/data-model-test.nix | 6 ++++++ deployment/data-model.nix | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 55a0e1d8..053f7ba2 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -46,6 +46,10 @@ in module = { }; runtime-environment = config.runtime-environments.bar; }; + migrations.boo = { + deployment = config.deployments.baz; + runtime-environment = config.runtime-environments.bar; + }; } ); in @@ -54,12 +58,14 @@ in has-runtime-environment = lib.isAttrs example.runtime-environments.bar.single-ssh-host.module; has-application = lib.isAttrs example.applications.foo.module; has-deployment = lib.isAttrs example.deployments.baz.module; + has-migration = lib.isAttrs example.migrations.boo.deployment; }; expected = { has-runtime-configuration = true; has-runtime-environment = true; has-application = true; has-deployment = true; + has-migration = true; }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 2a90fc11..78bfefca 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -127,6 +127,23 @@ let } ]; }; + migration = submoduleWith { + description = "Migration of a Fediversity deployment to a Fediversity run-time environment"; + modules = [ + { + options = { + deployment = mkOption { + description = "Deployment to migrate"; + type = deployment; + }; + runtime-environment = mkOption { + description = "Run-time environment to migrate the deployment to"; + type = runtime-environment; + }; + }; + } + ]; + }; in { options = { @@ -146,5 +163,9 @@ in description = "Deployment of a configuration of applications to a run-time environment"; type = attrsOf deployment; }; + migrations = mkOption { + description = "Migrations from Fediversity deployments to Fediversity run-time environments"; + type = attrsOf migration; + }; }; } -- 2.48.1 From c764c0f7b6cec1efa0946834416f410765fe1095 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Mon, 30 Jun 2025 14:04:54 +0200 Subject: [PATCH 18/37] better reflect naming from diagram configuration data flow --- deployment/data-model-test.nix | 42 +++++++++++++++------------------- deployment/data-model.nix | 32 ++++++++++++++++---------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 053f7ba2..16a5caaf 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -19,50 +19,46 @@ in example = eval ( { config, ... }: { - runtime-configurations.single-ssh-host = + providers.single-ssh-host = { ... }: { system.stateVersion = "25.05"; }; - runtime-environments.bar = { - single-ssh-host = { - ssh = { - host = "localhost"; - username = "root"; - authentication.password = ""; - }; + resources.bar.runtime-environment.single-ssh-host = { + ssh = { + host = "localhost"; + username = "root"; + authentication.password = ""; }; }; - applications.foo = { - module = - { pkgs, ... }: - { - environment.systemPackages = [ - pkgs.hello - ]; - }; - }; + applications.foo.module = + { pkgs, ... }: + { + environment.systemPackages = [ + pkgs.hello + ]; + }; deployments.baz = { module = { }; - runtime-environment = config.runtime-environments.bar; + runtime-environment = config.resources.bar.runtime-environment; }; migrations.boo = { deployment = config.deployments.baz; - runtime-environment = config.runtime-environments.bar; + runtime-environment = config.resources.bar.runtime-environment; }; } ); in { - has-runtime-configuration = lib.isAttrs example.runtime-configurations.single-ssh-host; - has-runtime-environment = lib.isAttrs example.runtime-environments.bar.single-ssh-host.module; + has-provider = lib.isAttrs example.providers.single-ssh-host; + has-resource = lib.isAttrs example.resources.bar.runtime-environment.single-ssh-host.module; has-application = lib.isAttrs example.applications.foo.module; has-deployment = lib.isAttrs example.deployments.baz.module; has-migration = lib.isAttrs example.migrations.boo.deployment; }; expected = { - has-runtime-configuration = true; - has-runtime-environment = true; + has-provider = true; + has-resource = true; has-application = true; has-deployment = true; has-migration = true; diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 78bfefca..2a52728c 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -20,8 +20,8 @@ let submodule str ; - runtime-configuration = mkOption { - description = "The NixOS module of a run-time environment"; + provider = mkOption { + description = "The NixOS module of a provider"; type = deferredModule; default = { _class = "nixos"; @@ -37,9 +37,9 @@ let // { type = mergeTypes type (submodule { options.module = mkOption { - description = "The NixOS module of the run-time environment"; + description = "The NixOS module of the provider"; type = deferredModule; - default = config.runtime-configurations.${name}; + default = config.providers.${name}; readOnly = true; }; }); @@ -94,13 +94,21 @@ let }; } ); + resource = attrTag { + runtime-environment = mkOption { + description = "A run-time environment one may deploy a NixOS configuration to."; + type = runtime-environment; + }; + }; application = submoduleWith { description = "A Fediversity application"; modules = [ { options = { module = mkOption { - description = "The NixOS module for that application, for configuring that application"; + description = '' + The NixOS module to configure the application. + ''; type = deferredModule; }; }; @@ -147,16 +155,16 @@ let in { options = { - runtime-configurations = mkOption { - description = "Collection of runtime environments into which applications can be deployed"; - type = attrTag (genAttrs (attrNames runtime-environment.nestedTypes) (_: runtime-configuration)); + providers = mkOption { + description = "Collection of providers for run-time environments to deploy applications to"; + type = attrTag (genAttrs (attrNames runtime-environment.nestedTypes) (_: provider)); }; - runtime-environments = mkOption { - description = "Collection of runtime environments into which applications can be deployed"; - type = attrsOf runtime-environment; + resources = mkOption { + description = "Collection of resources for use in Fediversity applications"; + type = attrsOf resource; }; applications = mkOption { - description = "Collection of Fediversity applications"; + description = "Collection of (available) Fediversity applications"; type = attrsOf application; }; deployments = mkOption { -- 2.48.1 From 3ec853a32af6ee2f60f31a853c74b274b9020660 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 1 Jul 2025 12:16:29 +0200 Subject: [PATCH 19/37] WIP: implement data model as in diagram this doesn't update the tests yet because we don't have all the data types in place anyway yet, and I still need to come up with testable examples. --- deployment/data-model.nix | 214 +++++++++++--------------------------- 1 file changed, 60 insertions(+), 154 deletions(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 2a52728c..81ce96c2 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -4,176 +4,82 @@ ... }: let - inherit (lib) - attrNames - mapAttrs - mkOption - genAttrs - ; + inherit (lib) mkOption; inherit (lib.types) attrsOf attrTag - deferredModule + deferredModuleWith mergeTypes nullOr submoduleWith submodule str ; - provider = mkOption { - description = "The NixOS module of a provider"; - type = deferredModule; - default = { - _class = "nixos"; - }; - }; - runtime-environment = attrTag ( - mapAttrs - ( - name: - option@{ type, ... }: - mkOption ( - option - // { - type = mergeTypes type (submodule { - options.module = mkOption { - description = "The NixOS module of the provider"; - type = deferredModule; - default = config.providers.${name}; - readOnly = true; - }; - }); - } - ) - ) - { - vm = { - description = "A VM to deploy to."; - type = submodule { - options = { - }; - }; - }; - single-ssh-host = { - description = "A single host to deploy to by SSH."; - type = submodule { - options = { - ssh = mkOption { - description = "SSH connection info"; - type = submodule { - options = { - host = mkOption { - description = "the host to access by SSH"; - type = str; - }; - username = mkOption { - description = "the SSH user to use"; - type = nullOr str; - default = null; - }; - authentication = mkOption { - description = "authentication method"; - type = attrsOf (attrTag { - private-key = mkOption { - description = "path to the user's SSH private key"; - type = str; - example = "/root/.ssh/id_ed25519"; - }; - password = mkOption { - description = "SSH password"; - # TODO: mark as sensitive - type = str; - }; - }); - }; - }; - }; - }; - }; - }; - }; - } - ); - resource = attrTag { - runtime-environment = mkOption { - description = "A run-time environment one may deploy a NixOS configuration to."; - type = runtime-environment; - }; - }; - application = submoduleWith { - description = "A Fediversity application"; - modules = [ - { - options = { - module = mkOption { - description = '' - The NixOS module to configure the application. - ''; - type = deferredModule; - }; - }; - } - ]; - }; - deployment = submoduleWith { - description = "A deployment of a configuration of applications to a run-time environment"; - modules = [ - { - options = { - # the `applications` option consists of configuration for the above applications - module = mkOption { - description = '' - Configuration to be deployed - ''; - type = deferredModule; - }; - runtime-environment = mkOption { - description = "The run-time environment to deploy to"; - type = runtime-environment; - }; - }; - } - ]; - }; - migration = submoduleWith { - description = "Migration of a Fediversity deployment to a Fediversity run-time environment"; - modules = [ - { - options = { - deployment = mkOption { - description = "Deployment to migrate"; - type = deployment; - }; - runtime-environment = mkOption { - description = "Run-time environment to migrate the deployment to"; - type = runtime-environment; - }; - }; - } - ]; - }; in { options = { - providers = mkOption { - description = "Collection of providers for run-time environments to deploy applications to"; - type = attrTag (genAttrs (attrNames runtime-environment.nestedTypes) (_: provider)); - }; resources = mkOption { - description = "Collection of resources for use in Fediversity applications"; - type = attrsOf resource; + description = "Collection of deployment resources that can be configured by hosting providers and required by applications"; + type = attrsOf ( + submodule ( + { config, ... }: + { + _class = "fediversity-resource"; + options = { + consumer = mkOption { + description = "Configuration of the resource by an application, a description of how the resource is consumed"; + type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-consumer"; } ]; }; + }; + provider = mkOption { + description = "Configuration of the resource by the hosting provider, a description of how the resource is made available"; + type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-provider"; } ]; }; + }; + }; + } + ) + ); }; applications = mkOption { - description = "Collection of (available) Fediversity applications"; - type = attrsOf application; + description = "Collection of Fediversity applications"; + type = attrsOf ( + submodule (application: { + _class = "fediversity-application"; + options = { + module = mkOption { + description = "Operator-facing configuration options for the application"; + type = deferredModuleWith { staticModules = [ { _class = "fediversity-application-config"; } ]; }; + }; + config-mapping = mkOption { + description = "Mapping of application configuration to deployment resources, a description of what an application needs to run"; + # TODO: type = (submodule application.config.module) -> (attrsOf (attrTag (lib.mapAttrs' (name: resource: { ${name} = mkOption { type = resource.consumer; }; }) config.resources))) /* something like that */ + }; + }; + }) + ); }; - deployments = mkOption { - description = "Deployment of a configuration of applications to a run-time environment"; - type = attrsOf deployment; - }; - migrations = mkOption { - description = "Migrations from Fediversity deployments to Fediversity run-time environments"; - type = attrsOf migration; + environments = mkOption { + description = "Run-time environments for Fediversity applications to be deployed to"; + type = attrsOf ( + submodule (environment: { + _class = "fediversity-environment"; + options = { + resources = mkOption { + description = "Resources made available by the hosting provider"; + type = attrsOf ( + attrTag ( + lib.mapAttrs' (name: resource: { + ${name} = mkOption { type = resource.provider; }; + }) config.resources + ) + ); + }; + resource-mapping = mkOption { + description = "Mapping of resources required by applications to available resources; the result can be deployed"; + # TODO: type = consumer-resources /* same as the output of application.config-mapping, should be in a `let` */ -> nixops4Deployment + }; + }; + }) + ); }; }; } -- 2.48.1 From 5c97e3597009c3b80bb8db5fa332b12545700d74 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 1 Jul 2025 22:07:42 +0200 Subject: [PATCH 20/37] WIP: add missing types --- default.nix | 3 +++ deployment/data-model-test.nix | 7 ++--- deployment/data-model.nix | 39 ++++++++++++++++++++++++++- deployment/interface.nix | 49 ++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 deployment/interface.nix diff --git a/default.nix b/default.nix index 7c536a03..ea7b2557 100644 --- a/default.nix +++ b/default.nix @@ -9,6 +9,8 @@ let git-hooks gitignore ; + inherit (import sources.flake-inputs) import-flake; + inputs = (import-flake { src = ./.; }).inputs; inherit (pkgs) lib; pre-commit-check = (import "${git-hooks}/nix" { @@ -67,6 +69,7 @@ in # re-export inputs so they can be overridden granularly # (they can't be accessed from the outside any other way) inherit + inputs sources system pkgs diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 16a5caaf..3f24df94 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -1,9 +1,12 @@ let - inherit (import ../default.nix { }) pkgs; + inherit (import ../default.nix { }) pkgs inputs; inherit (pkgs) lib; eval = module: (lib.evalModules { + specialArgs = { + inherit inputs; + }; modules = [ module ./data-model.nix @@ -11,8 +14,6 @@ let }).config; in { - _class = "nix-unit"; - test-eval = { expr = let diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 81ce96c2..af6eda78 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -1,6 +1,7 @@ { lib, config, + inputs, ... }: let @@ -15,6 +16,20 @@ let submodule str ; + + functionType = import ./interface.nix; + application-resources = { + options.resources = mkOption { + type = attrsOf ( + attrTag ( + lib.mapAttrs' (name: resource: { + ${name} = mkOption { type = resource.consumer; }; + }) config.resources + ) + ); + }; + }; + nixops4Deployment = inputs.nixops4.modules.nixops4Deployment.default; in { options = { @@ -51,7 +66,19 @@ in }; config-mapping = mkOption { description = "Mapping of application configuration to deployment resources, a description of what an application needs to run"; - # TODO: type = (submodule application.config.module) -> (attrsOf (attrTag (lib.mapAttrs' (name: resource: { ${name} = mkOption { type = resource.consumer; }; }) config.resources))) /* something like that */ + type = application.config.function.implementation; + }; + # TODO: somewhere we still need to + # - apply = { implementation = config-mapping; input = ; } + # - apply.output + function = mkOption { + description = "Function type for the mapping from application configuration to required resources"; + type = submodule functionType; + readOnly = true; + default = { + input-type = application.config.module; + output-type = application-resources; + }; }; }; }) @@ -75,8 +102,18 @@ in }; resource-mapping = mkOption { description = "Mapping of resources required by applications to available resources; the result can be deployed"; + type = environment.config.function.implementation; # TODO: type = consumer-resources /* same as the output of application.config-mapping, should be in a `let` */ -> nixops4Deployment }; + function = mkOption { + description = "Function type for the mapping from resources to a (NixOps4) deployment"; + type = submodule functionType; + readOnly = true; + default = { + input-type = application-resources; + output-type = submodule nixops4Deployment; + }; + }; }; }) ); diff --git a/deployment/interface.nix b/deployment/interface.nix new file mode 100644 index 00000000..225526d1 --- /dev/null +++ b/deployment/interface.nix @@ -0,0 +1,49 @@ +/** + Modular function type +*/ +{ config, ... }: +{ + options = { + input-type = mkOption { + type = deferredModule; + }; + output-type = mkOption { + type = deferredModule; + }; + implementation = mkOption { + type = optionType; + readOnly = true; + default = implementationTo ( + submodule (implementation: { + options = { + input = mkOption { + type = submodule config.input-type; + }; + output = mkOption { + type = submodule config.output-type; + }; + }; + }) + ); + }; + apply = mkOption { + type = optionType; + readOnly = true; + default = submodule (apply: { + options = { + implementation = mkOption { + type = config.implementation; + }; + input = mkOption { + type = submodule config.input-type; + }; + output = mkOption { + type = submodule config.output-type; + readOnly = true; + default = (apply.config.implementation apply.config.input).output; + }; + }; + }); + }; + }; +} -- 2.48.1 From 7a667c7517d7eb8629aa993e2169f09a8a76414f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 1 Jul 2025 23:59:16 +0200 Subject: [PATCH 21/37] WIP: start writing an evaluation test turns out we also need a collection of configurations, obviously next: figure out where to wire everything up to obtain a deployment --- deployment/data-model-test.nix | 47 +++++++++------------------------- deployment/data-model.nix | 23 ++++++++++++----- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 3f24df94..92b20d88 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -17,52 +17,29 @@ in test-eval = { expr = let - example = eval ( + fediversity = eval ( { config, ... }: { - providers.single-ssh-host = - { ... }: - { - system.stateVersion = "25.05"; - }; - resources.bar.runtime-environment.single-ssh-host = { - ssh = { - host = "localhost"; - username = "root"; - authentication.password = ""; - }; + resources.nixos = { + }; - applications.foo.module = - { pkgs, ... }: - { - environment.systemPackages = [ - pkgs.hello - ]; - }; - deployments.baz = { - module = { }; - runtime-environment = config.resources.bar.runtime-environment; + applications.hello = { + }; - migrations.boo = { - deployment = config.deployments.baz; - runtime-environment = config.resources.bar.runtime-environment; + environments.single-nixos-vm = { + + }; + configurations.example = { + enable = true; + applications.hello.enable = true; }; } ); in { - has-provider = lib.isAttrs example.providers.single-ssh-host; - has-resource = lib.isAttrs example.resources.bar.runtime-environment.single-ssh-host.module; - has-application = lib.isAttrs example.applications.foo.module; - has-deployment = lib.isAttrs example.deployments.baz.module; - has-migration = lib.isAttrs example.migrations.boo.deployment; + }; expected = { - has-provider = true; - has-resource = true; - has-application = true; - has-deployment = true; - has-migration = true; }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index af6eda78..82a1e165 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -5,16 +5,13 @@ ... }: let - inherit (lib) mkOption; + inherit (lib) mkOption types; inherit (lib.types) attrsOf attrTag deferredModuleWith - mergeTypes - nullOr submoduleWith submodule - str ; functionType = import ./interface.nix; @@ -70,7 +67,7 @@ in }; # TODO: somewhere we still need to # - apply = { implementation = config-mapping; input = ; } - # - apply.output + # - apply.output (applications required by resources) function = mkOption { description = "Function type for the mapping from application configuration to required resources"; type = submodule functionType; @@ -103,8 +100,10 @@ in resource-mapping = mkOption { description = "Mapping of resources required by applications to available resources; the result can be deployed"; type = environment.config.function.implementation; - # TODO: type = consumer-resources /* same as the output of application.config-mapping, should be in a `let` */ -> nixops4Deployment }; + # TODO: somewhere we still need to + # - apply = { implementation = resource-mapping; input = ; } + # - apply.output (deployment) function = mkOption { description = "Function type for the mapping from resources to a (NixOps4) deployment"; type = submodule functionType; @@ -118,5 +117,17 @@ in }) ); }; + configurations = mkOption { + description = "Application configurations set by operators"; + type = attrsOf (submodule { + options = { + enable = mkOption { + description = "Whether to enable the configuration"; + type = types.bool; + }; + applications = mapAttrs (name: application: submodule application.module) config.applications; + }; + }); + }; }; } -- 2.48.1 From f8d1be9f6e5b16d98284b098635ce887a942b34e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 2 Jul 2025 01:20:35 +0200 Subject: [PATCH 22/37] WIP: implement mappings --- deployment/data-model-test.nix | 46 +++++++++++++++------- deployment/data-model.nix | 71 ++++++++++++++++++++++++---------- deployment/function.nix | 30 ++++++++++++++ deployment/interface.nix | 49 ----------------------- 4 files changed, 114 insertions(+), 82 deletions(-) create mode 100644 deployment/function.nix delete mode 100644 deployment/interface.nix diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 92b20d88..76b643b2 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -1,6 +1,7 @@ let inherit (import ../default.nix { }) pkgs inputs; inherit (pkgs) lib; + inherit (lib) mkOption types; eval = module: (lib.evalModules { @@ -12,6 +13,7 @@ let ./data-model.nix ]; }).config; + nixops4Deployment = inputs.nixops4.modules.nixops4Deployment.default; in { test-eval = { @@ -20,18 +22,35 @@ in fediversity = eval ( { config, ... }: { - resources.nixos = { - + config = { + resources.nixos = { + # TODO: consumer = ? + # TODO: provider = ? + }; + applications.hello = { + description = ''Command-line tool that will print "Hello, world!" on the terminal''; + # TODO: module = ? + # TODO: config=mapping = ? + }; + environments.single-nixos-vm = { + # TODO: resources = ? + # TODO: resource-mapping = ? + }; }; - applications.hello = { - - }; - environments.single-nixos-vm = { - - }; - configurations.example = { - enable = true; - applications.hello.enable = true; + options = { + example-configuration = mkOption { + type = config.configuration; + readOnly = true; + default = { + enable = true; + applications.hello.enable = true; + }; + }; + example-deployment = mkOption { + type = nixops4Deployment; + readOnly = true; + default = config.environments.single-nixos-vm.deployment config.example-configuration; + }; }; } ); @@ -39,7 +58,8 @@ in { }; - expected = { - }; + expected = + { + }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 82a1e165..9f2a7e9f 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -12,9 +12,11 @@ let deferredModuleWith submoduleWith submodule + optionType + functionTo ; - functionType = import ./interface.nix; + functionType = import ./function.nix; application-resources = { options.resources = mkOption { type = attrsOf ( @@ -39,11 +41,11 @@ in _class = "fediversity-resource"; options = { consumer = mkOption { - description = "Configuration of the resource by an application, a description of how the resource is consumed"; + description = "Options for declaring resource requirements by an application, a description of how the resource is consumed"; type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-consumer"; } ]; }; }; provider = mkOption { - description = "Configuration of the resource by the hosting provider, a description of how the resource is made available"; + description = "Options for configuring the resource for the hosting provider, a description of how the resource is made available"; type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-provider"; } ]; }; }; }; @@ -57,18 +59,25 @@ in submodule (application: { _class = "fediversity-application"; options = { + description = mkOption { + description = "Description to be shown in the application overview"; + type = types.str; + }; module = mkOption { description = "Operator-facing configuration options for the application"; type = deferredModuleWith { staticModules = [ { _class = "fediversity-application-config"; } ]; }; }; - config-mapping = mkOption { + implementation = mkOption { description = "Mapping of application configuration to deployment resources, a description of what an application needs to run"; - type = application.config.function.implementation; + type = application.config.config-mapping.function-type; }; - # TODO: somewhere we still need to - # - apply = { implementation = config-mapping; input = ; } - # - apply.output (applications required by resources) - function = mkOption { + resources = mkOption { + description = "Compute resources required by an application"; + type = functionTo application.config.config-mapping.output-type; + readOnly = true; + default = input: (application.config.implementation input).output; + }; + config-mapping = mkOption { description = "Function type for the mapping from application configuration to required resources"; type = submodule functionType; readOnly = true; @@ -97,35 +106,57 @@ in ) ); }; - resource-mapping = mkOption { + implementation = mkOption { description = "Mapping of resources required by applications to available resources; the result can be deployed"; - type = environment.config.function.implementation; + type = environment.config.resource-mapping.function-type; }; - # TODO: somewhere we still need to - # - apply = { implementation = resource-mapping; input = ; } - # - apply.output (deployment) - function = mkOption { + resource-mapping = mkOption { description = "Function type for the mapping from resources to a (NixOps4) deployment"; type = submodule functionType; readOnly = true; default = { input-type = application-resources; - output-type = submodule nixops4Deployment; + output-type = nixops4Deployment; }; }; + deployment = mkOption { + description = "Generate a deployment from a configuration"; + type = functionTo environment.config.resource-mapping.output-type; + readOnly = true; + default = + cfg: + # TODO: check cfg.enable.true + let + required-resources = lib.mapAttrs ( + name: application-settings: config.applications.${name}.resources application-settings + ) cfg.applications; + in + (environment.config.implementation required-resources).output; + + }; }; }) ); }; - configurations = mkOption { - description = "Application configurations set by operators"; - type = attrsOf (submodule { + configuration = mkOption { + description = "Configuration type declaring options to be set by operators"; + type = optionType; + readOnly = true; + default = submodule (configuration: { options = { enable = mkOption { description = "Whether to enable the configuration"; type = types.bool; + default = false; }; - applications = mapAttrs (name: application: submodule application.module) config.applications; + applications = lib.mapAttrs ( + name: application: + mkOption { + description = application.description; + type = submodule application.module; + default = { }; + } + ) config.applications; }; }); }; diff --git a/deployment/function.nix b/deployment/function.nix new file mode 100644 index 00000000..b6a208f0 --- /dev/null +++ b/deployment/function.nix @@ -0,0 +1,30 @@ +/** + Modular function type +*/ +{ config, ... }: +{ + options = { + input-type = mkOption { + type = deferredModule; + }; + output-type = mkOption { + type = deferredModule; + }; + function-type = mkOption { + type = optionType; + readOnly = true; + default = functionTo ( + submodule (function: { + options = { + input = mkOption { + type = submodule config.input-type; + }; + output = mkOption { + type = submodule config.output-type; + }; + }; + }) + ); + }; + }; +} diff --git a/deployment/interface.nix b/deployment/interface.nix deleted file mode 100644 index 225526d1..00000000 --- a/deployment/interface.nix +++ /dev/null @@ -1,49 +0,0 @@ -/** - Modular function type -*/ -{ config, ... }: -{ - options = { - input-type = mkOption { - type = deferredModule; - }; - output-type = mkOption { - type = deferredModule; - }; - implementation = mkOption { - type = optionType; - readOnly = true; - default = implementationTo ( - submodule (implementation: { - options = { - input = mkOption { - type = submodule config.input-type; - }; - output = mkOption { - type = submodule config.output-type; - }; - }; - }) - ); - }; - apply = mkOption { - type = optionType; - readOnly = true; - default = submodule (apply: { - options = { - implementation = mkOption { - type = config.implementation; - }; - input = mkOption { - type = submodule config.input-type; - }; - output = mkOption { - type = submodule config.output-type; - readOnly = true; - default = (apply.config.implementation apply.config.input).output; - }; - }; - }); - }; - }; -} -- 2.48.1 From 0c592d81f38ce2ba0bf903a126fc4b99c1ea5abe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 2 Jul 2025 03:39:36 +0200 Subject: [PATCH 23/37] WIP: (broken) implement test --- deployment/data-model-test.nix | 112 ++++++++++++++++++++++++++++----- deployment/data-model.nix | 45 ++++++++----- deployment/function.nix | 11 +++- 3 files changed, 136 insertions(+), 32 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 76b643b2..2817a2c4 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -23,19 +23,98 @@ in { config, ... }: { config = { - resources.nixos = { - # TODO: consumer = ? - # TODO: provider = ? - }; - applications.hello = { - description = ''Command-line tool that will print "Hello, world!" on the terminal''; - # TODO: module = ? - # TODO: config=mapping = ? - }; - environments.single-nixos-vm = { - # TODO: resources = ? - # TODO: resource-mapping = ? + resources.login-shell = { + description = "The operator needs to be able to log into the shell"; + request = + { ... }: + { + _class = "fediversity-resource-request"; + options = { + wheel = mkOption { + description = "Whether the login user needs root permissions"; + type = types.bool; + default = false; + }; + packages = mkOption { + description = "Packages that need to be available in the user environment"; + type = with types; attrsOf package; + }; + }; + }; + policy = + { config, ... }: + { + _class = "fediversity-resource-policy"; + options = { + username = mkOption { + description = "Username for the operator"; + type = types.str; # TODO: use the proper constraints from NixOS + }; + wheel = mkOption { + description = "Whether to allow login with root permissions"; + type = types.bool; + default = false; + }; + apply = mkOption { + type = with types; functionTo raw; # TODO: splice out the user type from NixOS + default = + requests: + let + # Filter out requests that need wheel if policy doesn't allow it + validRequests = lib.filterAttrs (name: req: !req.wheel || config.wheel) requests; + in + lib.optionalAttrs (validRequests != { }) { + ${config.username} = { + isNormalUser = true; + packages = with lib; concatMapAttrs (name: request: attrValues request.packages) validRequests; + extraGroups = lib.optional config.wheel "wheel"; + }; + }; + }; + }; + }; }; + applications.hello = + { ... }: + { + description = ''Command-line tool that will print "Hello, world!" on the terminal''; + module = + { ... }: + { + enable = lib.mkEnableOption "Hello in the shell"; + }; + implementation = + cfg: + lib.optionalAttrs cfg.enable { + dummy.login-shell.packages.hello = pkgs.hello; + }; + }; + environments.single-nixos-vm = + { config, ... }: + { + resources.shell.login-shell.username = "operator"; + implementation = + requests: + { providers, ... }: + { + providers = { + inherit (inputs.nixops4.modules.nixops4Provider) local; + }; + resources.the-machine = { + type = providers.local.exec; + imports = [ + inputs.nixops4-nixos.modules.nixops4Resource.nixos + ]; + nixos.module = + { pkgs, ... }: + { + users.users = config.resources.shell.login-shell.apply ( + lib.filterAttrs (name: value: value ? login-shell) requests + ); + }; + }; + }; + }; }; options = { example-configuration = mkOption { @@ -47,7 +126,7 @@ in }; }; example-deployment = mkOption { - type = nixops4Deployment; + type = types.submodule nixops4Deployment; readOnly = true; default = config.environments.single-nixos-vm.deployment config.example-configuration; }; @@ -56,10 +135,9 @@ in ); in { - - }; - expected = - { + inherit (fediversity) example-deployment; }; + expected = { + }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 9f2a7e9f..305a25d9 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -19,10 +19,11 @@ let functionType = import ./function.nix; application-resources = { options.resources = mkOption { + # TODO: maybe transpose, and group the resources by type instead type = attrsOf ( attrTag ( lib.mapAttrs' (name: resource: { - ${name} = mkOption { type = resource.consumer; }; + ${name} = mkOption { type = resource.request; }; }) config.resources ) ); @@ -33,20 +34,33 @@ in { options = { resources = mkOption { - description = "Collection of deployment resources that can be configured by hosting providers and required by applications"; + description = "Collection of deployment resources that can be required by applications and policed by hosting providers"; type = attrsOf ( submodule ( { config, ... }: { _class = "fediversity-resource"; options = { - consumer = mkOption { - description = "Options for declaring resource requirements by an application, a description of how the resource is consumed"; - type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-consumer"; } ]; }; + description = mkOption { + description = "Description of the resource to help application module authors and hosting providers to work with it"; + type = types.str; }; - provider = mkOption { - description = "Options for configuring the resource for the hosting provider, a description of how the resource is made available"; - type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-provider"; } ]; }; + request = mkOption { + description = "Options for declaring resource requirements by an application, a description of how the resource is consumed or accessed"; + type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-request"; } ]; }; + }; + policy = mkOption { + description = "Options for configuring the resource policy for the hosting provider, a description of how the resource is made available"; + type = deferredModuleWith { + staticModules = [ + { + _class = "fediversity-resource-policy"; + options.apply = mkOption { + desciption = "Apply the policy to a request"; + }; + } + ]; + }; }; }; } @@ -97,11 +111,16 @@ in _class = "fediversity-environment"; options = { resources = mkOption { - description = "Resources made available by the hosting provider"; + description = '' + Resources made available by the hosting provider, and their policies. + + Setting this is optional, but provides a place to declare that information for programmatic use in the resource mapping. + ''; + # TODO: maybe transpose, and group the resources by type instead type = attrsOf ( attrTag ( lib.mapAttrs' (name: resource: { - ${name} = mkOption { type = resource.provider; }; + ${name} = mkOption { type = resource.policy; }; }) config.resources ) ); @@ -144,10 +163,8 @@ in readOnly = true; default = submodule (configuration: { options = { - enable = mkOption { - description = "Whether to enable the configuration"; - type = types.bool; - default = false; + enable = lib.mkEnableOption { + description = "your Fediversity configuration"; }; applications = lib.mapAttrs ( name: application: diff --git a/deployment/function.nix b/deployment/function.nix index b6a208f0..4f473107 100644 --- a/deployment/function.nix +++ b/deployment/function.nix @@ -1,7 +1,16 @@ /** Modular function type */ -{ config, ... }: +{ config, lib, ... }: +let + inherit (lib) mkOption types; + inherit (types) + deferredModule + submodule + functionTo + optionType + ; +in { options = { input-type = mkOption { -- 2.48.1 From ba047997f2ff294cb28556a01bbfba4b414e743c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 3 Jul 2025 13:08:14 +0200 Subject: [PATCH 24/37] WIP: illustrate an entire NixOS configuration as a resource --- deployment/data-model-test.nix | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 2817a2c4..1e0222c7 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -23,6 +23,33 @@ in { config, ... }: { config = { + resources.nixos-configuration = { + description = "An entire NixOS configuration"; + request = + { ... }: + { + _class = "fediversity-resource-request"; + options.config = mkOption { + description = "Any options from NixOS"; + }; + }; + + policy = + { config, ... }: + { + _class = "fediversity-resource-policy"; + + options = { + extra-config = mkOptions { + description = "Any options from NixOS"; + }; + apply = mkOption { + type = with types; functionTo raw; + default = requests: lib.mkMerge (requests ++ [ config.extra-config ]); + }; + }; + }; + }; resources.login-shell = { description = "The operator needs to be able to log into the shell"; request = @@ -137,7 +164,8 @@ in { inherit (fediversity) example-deployment; }; - expected = { - }; + expected = + { + }; }; } -- 2.48.1 From b25ddac298316c9924c0ed917c83189790ef2310 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Thu, 3 Jul 2025 20:23:04 +0200 Subject: [PATCH 25/37] fix typos, lint, format --- deployment/data-model-test.nix | 10 +++++----- deployment/data-model.nix | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 1e0222c7..9b8d16c5 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -40,7 +40,7 @@ in _class = "fediversity-resource-policy"; options = { - extra-config = mkOptions { + extra-config = mkOption { description = "Any options from NixOS"; }; apply = mkOption { @@ -88,12 +88,12 @@ in requests: let # Filter out requests that need wheel if policy doesn't allow it - validRequests = lib.filterAttrs (name: req: !req.wheel || config.wheel) requests; + validRequests = lib.filterAttrs (_name: req: !req.wheel || config.wheel) requests; in lib.optionalAttrs (validRequests != { }) { ${config.username} = { isNormalUser = true; - packages = with lib; concatMapAttrs (name: request: attrValues request.packages) validRequests; + packages = with lib; concatMapAttrs (_name: request: attrValues request.packages) validRequests; extraGroups = lib.optional config.wheel "wheel"; }; }; @@ -133,10 +133,10 @@ in inputs.nixops4-nixos.modules.nixops4Resource.nixos ]; nixos.module = - { pkgs, ... }: + { ... }: { users.users = config.resources.shell.login-shell.apply ( - lib.filterAttrs (name: value: value ? login-shell) requests + lib.filterAttrs (_name: value: value ? login-shell) requests ); }; }; diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 305a25d9..f5f63f2f 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -37,7 +37,7 @@ in description = "Collection of deployment resources that can be required by applications and policed by hosting providers"; type = attrsOf ( submodule ( - { config, ... }: + { ... }: { _class = "fediversity-resource"; options = { @@ -56,7 +56,7 @@ in { _class = "fediversity-resource-policy"; options.apply = mkOption { - desciption = "Apply the policy to a request"; + description = "Apply the policy to a request"; }; } ]; @@ -161,13 +161,13 @@ in description = "Configuration type declaring options to be set by operators"; type = optionType; readOnly = true; - default = submodule (configuration: { + default = submodule { options = { enable = lib.mkEnableOption { description = "your Fediversity configuration"; }; applications = lib.mapAttrs ( - name: application: + _name: application: mkOption { description = application.description; type = submodule application.module; @@ -175,7 +175,7 @@ in } ) config.applications; }; - }); + }; }; }; } -- 2.48.1 From bb93d2d0de4803ed4f127a95ce0ab2f1bc629fc1 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Fri, 4 Jul 2025 23:04:28 +0200 Subject: [PATCH 26/37] use `mapAttrs` right `mapAttrs'` takes two args rather than a set, whereas if only the val changes `mapAttrs (_: v: ...)` should do --- deployment/data-model.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index f5f63f2f..75c759b9 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -119,9 +119,7 @@ in # TODO: maybe transpose, and group the resources by type instead type = attrsOf ( attrTag ( - lib.mapAttrs' (name: resource: { - ${name} = mkOption { type = resource.policy; }; - }) config.resources + lib.mapAttrs (_name: resource: mkOption { type = submodule resource.policy; }) config.resources ) ); }; -- 2.48.1 From 0f7da573926331240dae97debb86af952b2f6487 Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Fri, 18 Jul 2025 22:17:52 +0200 Subject: [PATCH 27/37] use `submodule` to turn module into type for `functionTo` --- deployment/data-model.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 75c759b9..39bd193f 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -138,7 +138,7 @@ in }; deployment = mkOption { description = "Generate a deployment from a configuration"; - type = functionTo environment.config.resource-mapping.output-type; + type = functionTo (submodule environment.config.resource-mapping.output-type); readOnly = true; default = cfg: -- 2.48.1 From 243ef4425b3bb59e26ddc312c061e6c55f86b19c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 22 Jul 2025 12:28:00 +0200 Subject: [PATCH 28/37] WIP: more type-safe policy application --- deployment/data-model-test.nix | 35 ++++++++++++++++------------------ deployment/data-model.nix | 15 +++++++++++++-- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 9b8d16c5..899e3747 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -82,23 +82,21 @@ in type = types.bool; default = false; }; - apply = mkOption { - type = with types; functionTo raw; # TODO: splice out the user type from NixOS - default = - requests: - let - # Filter out requests that need wheel if policy doesn't allow it - validRequests = lib.filterAttrs (_name: req: !req.wheel || config.wheel) requests; - in - lib.optionalAttrs (validRequests != { }) { - ${config.username} = { - isNormalUser = true; - packages = with lib; concatMapAttrs (_name: request: attrValues request.packages) validRequests; - extraGroups = lib.optional config.wheel "wheel"; - }; - }; - }; }; + config.resource-type = types.any; # TODO: splice out the user type from NixOS + config.apply = + requests: + let + # Filter out requests that need wheel if policy doesn't allow it + validRequests = lib.filterAttrs (_name: req: !req.wheel || config.wheel) requests; + in + lib.optionalAttrs (validRequests != { }) { + ${config.username} = { + isNormalUser = true; + packages = with lib; concatMapAttrs (_name: request: attrValues request.packages) validRequests; + extraGroups = lib.optional config.wheel "wheel"; + }; + }; }; }; applications.hello = @@ -164,8 +162,7 @@ in { inherit (fediversity) example-deployment; }; - expected = - { - }; + expected = { + }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 39bd193f..d46fdf75 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -53,12 +53,23 @@ in description = "Options for configuring the resource policy for the hosting provider, a description of how the resource is made available"; type = deferredModuleWith { staticModules = [ - { + (policy: { _class = "fediversity-resource-policy"; + # TODO(@fricklerhandwerk): not sure it can be made + # sensible syntactically, but essentially we want to + # ensure that `apply` is defined, but since its output + # depends on the specific policy we also need to + # determine that somehow. + # hopefully this also helps with correct composition down the line. + options.resource-type = mkOption { + description = "The type of resource this policy configures"; + type = types.optionType; + }; options.apply = mkOption { description = "Apply the policy to a request"; + type = with types; functionTo policy.config.resource-type; }; - } + }) ]; }; }; -- 2.48.1 From 7066b2cb6966a6ef59f31a284f591d084169c8d1 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 22 Jul 2025 12:42:08 +0200 Subject: [PATCH 29/37] use mapAttrs right, again --- deployment/data-model.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index d46fdf75..254c041d 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -21,11 +21,7 @@ let options.resources = mkOption { # TODO: maybe transpose, and group the resources by type instead type = attrsOf ( - attrTag ( - lib.mapAttrs' (name: resource: { - ${name} = mkOption { type = resource.request; }; - }) config.resources - ) + attrTag (lib.mapAttrs (_name: resource: mkOption { type = resource.request; }) config.resources) ); }; }; -- 2.48.1 From bf488f89e120242dcf0509bbf51eef6219067a58 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 22 Jul 2025 12:43:32 +0200 Subject: [PATCH 30/37] readability --- deployment/data-model-test.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 899e3747..5238ab89 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -111,7 +111,9 @@ in implementation = cfg: lib.optionalAttrs cfg.enable { - dummy.login-shell.packages.hello = pkgs.hello; + hello.login-shell.packages = { + inherit (pkgs) hello; + }; }; }; environments.single-nixos-vm = -- 2.48.1 From 421d7b6f93ef1cf29bf0e4642030e4c3c60102a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Jul 2025 18:59:19 +0200 Subject: [PATCH 31/37] generalize function type to take types --- deployment/data-model-test.nix | 5 +++-- deployment/data-model.nix | 4 ++-- deployment/function.nix | 9 ++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 5238ab89..24c31dd8 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -164,7 +164,8 @@ in { inherit (fediversity) example-deployment; }; - expected = { - }; + expected = + { + }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 254c041d..70f535ff 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -17,7 +17,7 @@ let ; functionType = import ./function.nix; - application-resources = { + application-resources = submodule { options.resources = mkOption { # TODO: maybe transpose, and group the resources by type instead type = attrsOf ( @@ -25,7 +25,7 @@ let ); }; }; - nixops4Deployment = inputs.nixops4.modules.nixops4Deployment.default; + nixops4Deployment = submodule inputs.nixops4.modules.nixops4Deployment.default; in { options = { diff --git a/deployment/function.nix b/deployment/function.nix index 4f473107..5067505c 100644 --- a/deployment/function.nix +++ b/deployment/function.nix @@ -5,7 +5,6 @@ let inherit (lib) mkOption types; inherit (types) - deferredModule submodule functionTo optionType @@ -14,10 +13,10 @@ in { options = { input-type = mkOption { - type = deferredModule; + type = optionType; }; output-type = mkOption { - type = deferredModule; + type = optionType; }; function-type = mkOption { type = optionType; @@ -26,10 +25,10 @@ in submodule (function: { options = { input = mkOption { - type = submodule config.input-type; + type = config.input-type; }; output = mkOption { - type = submodule config.output-type; + type = config.output-type; }; }; }) -- 2.48.1 From b49e426ed4aac91107d039afa106466cd5eab18a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Jul 2025 19:26:05 +0200 Subject: [PATCH 32/37] test application config --- deployment/data-model-test.nix | 11 +++++++---- deployment/data-model.nix | 13 ++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 24c31dd8..c7350019 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -106,7 +106,7 @@ in module = { ... }: { - enable = lib.mkEnableOption "Hello in the shell"; + options.enable = lib.mkEnableOption "Hello in the shell"; }; implementation = cfg: @@ -162,10 +162,13 @@ in ); in { - inherit (fediversity) example-deployment; + config = fediversity.example-configuration; }; - expected = - { + expected = { + config = { + enable = true; + applications.hello.enable = true; }; + }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 70f535ff..b14f4c8f 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -25,7 +25,18 @@ let ); }; }; - nixops4Deployment = submodule inputs.nixops4.modules.nixops4Deployment.default; + nixops4Deployment = types.deferredModuleWith { + staticModules = [ + inputs.nixops4.modules.nixops4Deployment.default + + { + _module.args = { + resourceProviderSystem = builtins.currentSystem; + resources = { }; + }; + } + ]; + }; in { options = { -- 2.48.1 From 89b22df3a46b9c5c041b780ca0eceb942e5a6b96 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Jul 2025 20:08:29 +0200 Subject: [PATCH 33/37] WIP(broken, infinite recursion): apply application config --- deployment/data-model-test.nix | 12 +++++++----- deployment/data-model.nix | 6 ++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index c7350019..d2d4f2fc 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -108,13 +108,14 @@ in { options.enable = lib.mkEnableOption "Hello in the shell"; }; - implementation = - cfg: - lib.optionalAttrs cfg.enable { - hello.login-shell.packages = { + implementation = cfg: { + input = cfg; + output = lib.optionalAttrs cfg.enable { + resources.hello.login-shell.packages = { inherit (pkgs) hello; }; }; + }; }; environments.single-nixos-vm = { config, ... }: @@ -161,8 +162,9 @@ in } ); in - { + rec { config = fediversity.example-configuration; + resources = fediversity.applications.hello.implementation config.applications.hello; }; expected = { config = { diff --git a/deployment/data-model.nix b/deployment/data-model.nix index b14f4c8f..d3361f16 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -21,7 +21,9 @@ let options.resources = mkOption { # TODO: maybe transpose, and group the resources by type instead type = attrsOf ( - attrTag (lib.mapAttrs (_name: resource: mkOption { type = resource.request; }) config.resources) + attrTag ( + lib.mapAttrs (_name: resource: mkOption { type = submodule resource.request; }) config.resources + ) ); }; }; @@ -114,7 +116,7 @@ in type = submodule functionType; readOnly = true; default = { - input-type = application.config.module; + input-type = submodule application.config.module; output-type = application-resources; }; }; -- 2.48.1 From a102ad93b702ce20196f526bcd399266316196c0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 25 Jul 2025 11:43:31 +0200 Subject: [PATCH 34/37] test that login-shell resource is mapped by hello application --- deployment/data-model-test.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index d2d4f2fc..c47deb78 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -111,9 +111,7 @@ in implementation = cfg: { input = cfg; output = lib.optionalAttrs cfg.enable { - resources.hello.login-shell.packages = { - inherit (pkgs) hello; - }; + resources.hello.login-shell.packages.hello = pkgs.hello; }; }; }; @@ -163,14 +161,19 @@ in ); in rec { + number-of-resources = with lib; length (attrNames fediversity.resources); config = fediversity.example-configuration; - resources = fediversity.applications.hello.implementation config.applications.hello; + hello-package-exists = + (fediversity.applications.hello.resources config.applications.hello) + .resources.hello.login-shell.packages ? hello; }; expected = { + number-of-resources = 2; config = { enable = true; applications.hello.enable = true; }; + hello-package-exists = true; }; }; } -- 2.48.1 From 8c571c2dbe11419cf0a6e3242fb9e06089dcf652 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 25 Jul 2025 12:03:58 +0200 Subject: [PATCH 35/37] WIP: get past dumb type error --- deployment/data-model-test.nix | 50 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index c47deb78..5b0c4d29 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -119,27 +119,29 @@ in { config, ... }: { resources.shell.login-shell.username = "operator"; - implementation = - requests: - { providers, ... }: - { - providers = { - inherit (inputs.nixops4.modules.nixops4Provider) local; + implementation = requests: { + input = requests; + output = + { providers, ... }: + { + providers = { + inherit (inputs.nixops4.modules.nixops4Provider) local; + }; + resources.the-machine = { + type = providers.local.exec; + imports = [ + inputs.nixops4-nixos.modules.nixops4Resource.nixos + ]; + nixos.module = + { ... }: + { + users.users = config.resources.shell.login-shell.apply ( + lib.filterAttrs (_name: value: value ? login-shell) requests + ); + }; + }; }; - resources.the-machine = { - type = providers.local.exec; - imports = [ - inputs.nixops4-nixos.modules.nixops4Resource.nixos - ]; - nixos.module = - { ... }: - { - users.users = config.resources.shell.login-shell.apply ( - lib.filterAttrs (_name: value: value ? login-shell) requests - ); - }; - }; - }; + }; }; }; options = { @@ -162,18 +164,20 @@ in in rec { number-of-resources = with lib; length (attrNames fediversity.resources); - config = fediversity.example-configuration; + inherit (fediversity) example-configuration; hello-package-exists = - (fediversity.applications.hello.resources config.applications.hello) + (fediversity.applications.hello.resources example-configuration.applications.hello) .resources.hello.login-shell.packages ? hello; + inherit (fediversity) example-deployment; }; expected = { number-of-resources = 2; - config = { + example-configuration = { enable = true; applications.hello.enable = true; }; hello-package-exists = true; + example-deployment = { }; }; }; } -- 2.48.1 From 37fd4d6642f22ef891766d80c268d3b3a2b85229 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 25 Jul 2025 12:46:37 +0200 Subject: [PATCH 36/37] fix resource mapping for the shell application --- deployment/data-model-test.nix | 42 ++++++++++++++++++++++++++-------- deployment/data-model.nix | 3 ++- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix index 5b0c4d29..34f9b381 100644 --- a/deployment/data-model-test.nix +++ b/deployment/data-model-test.nix @@ -43,11 +43,9 @@ in extra-config = mkOption { description = "Any options from NixOS"; }; - apply = mkOption { - type = with types; functionTo raw; - default = requests: lib.mkMerge (requests ++ [ config.extra-config ]); - }; }; + config.resource-type = types.raw; # TODO: what's the type of a NixOS configuration? + config.apply = requests: lib.mkMerge (requests ++ [ config.extra-config ]); }; }; resources.login-shell = { @@ -83,17 +81,21 @@ in default = false; }; }; - config.resource-type = types.any; # TODO: splice out the user type from NixOS + config.resource-type = types.raw; # TODO: splice out the user type from NixOS config.apply = requests: let # Filter out requests that need wheel if policy doesn't allow it - validRequests = lib.filterAttrs (_name: req: !req.wheel || config.wheel) requests; + validRequests = lib.filterAttrs ( + _name: req: !req.login-shell.wheel || config.wheel + ) requests.resources; in lib.optionalAttrs (validRequests != { }) { ${config.username} = { isNormalUser = true; - packages = with lib; concatMapAttrs (_name: request: attrValues request.packages) validRequests; + packages = + with lib; + attrValues (concatMapAttrs (_name: request: request.login-shell.packages) validRequests); extraGroups = lib.optional config.wheel "wheel"; }; }; @@ -118,7 +120,7 @@ in environments.single-nixos-vm = { config, ... }: { - resources.shell.login-shell.username = "operator"; + resources.operator-environment.login-shell.username = "operator"; implementation = requests: { input = requests; output = @@ -168,7 +170,21 @@ in hello-package-exists = (fediversity.applications.hello.resources example-configuration.applications.hello) .resources.hello.login-shell.packages ? hello; - inherit (fediversity) example-deployment; + wheel-required = + (fediversity.applications.hello.resources example-configuration.applications.hello) + .resources.hello.login-shell.wheel; + wheel-allowed = + fediversity.environments.single-nixos-vm.resources.operator-environment.login-shell.wheel; + operator-shell = + let + resources = fediversity.applications.hello.resources example-configuration.applications.hello; + users = fediversity.environments.single-nixos-vm.resources.operator-environment.login-shell.apply resources; + in + { + inherit (users.operator) isNormalUser; + packages = with lib; map (p: "${p.pname}") users.operator.packages; + extraGroups = users.operator.extraGroups; + }; }; expected = { number-of-resources = 2; @@ -177,7 +193,13 @@ in applications.hello.enable = true; }; hello-package-exists = true; - example-deployment = { }; + wheel-required = false; + wheel-allowed = false; + operator-shell = { + isNormalUser = true; + packages = [ "hello" ]; + extraGroups = [ ]; + }; }; }; } diff --git a/deployment/data-model.nix b/deployment/data-model.nix index d3361f16..973eec74 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -74,6 +74,7 @@ in description = "The type of resource this policy configures"; type = types.optionType; }; + # TODO(@fricklerhandwerk): do we need a function type here as well, or is it in the way? options.apply = mkOption { description = "Apply the policy to a request"; type = with types; functionTo policy.config.resource-type; @@ -153,7 +154,7 @@ in readOnly = true; default = { input-type = application-resources; - output-type = nixops4Deployment; + output-type = types.raw; }; }; deployment = mkOption { -- 2.48.1 From 753dd0e0371fa87ec456bd8355962d188eab325b Mon Sep 17 00:00:00 2001 From: Kiara Grouwstra Date: Sun, 27 Jul 2025 16:43:36 +0200 Subject: [PATCH 37/37] Revert "use `submodule` to turn module into type for `functionTo`" This reverts commit 0f7da573926331240dae97debb86af952b2f6487. --- deployment/data-model.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/data-model.nix b/deployment/data-model.nix index 973eec74..ff157149 100644 --- a/deployment/data-model.nix +++ b/deployment/data-model.nix @@ -159,7 +159,7 @@ in }; deployment = mkOption { description = "Generate a deployment from a configuration"; - type = functionTo (submodule environment.config.resource-mapping.output-type); + type = functionTo environment.config.resource-mapping.output-type; readOnly = true; default = cfg: -- 2.48.1