diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 18925ab8..f7f0a438 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -15,6 +15,12 @@ jobs: - uses: actions/checkout@v4 - run: nix-build -A tests + check-data-model: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix-shell --run 'nix-unit ./deployment/data-model-test.nix' + check-peertube: runs-on: native steps: @@ -38,3 +44,9 @@ jobs: steps: - uses: actions/checkout@v4 - run: nix build .#checks.x86_64-linux.deployment-cli -L + + check-deployment-panel: + runs-on: native + steps: + - uses: actions/checkout@v4 + - run: nix build .#checks.x86_64-linux.deployment-panel -L diff --git a/.forgejo/workflows/update.yaml b/.forgejo/workflows/update.yaml index d76c9622..ac9a17f4 100644 --- a/.forgejo/workflows/update.yaml +++ b/.forgejo/workflows/update.yaml @@ -7,15 +7,16 @@ on: jobs: lockfile: - runs-on: ubuntu-latest + runs-on: native steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Nix - uses: cachix/install-nix-action@v31 - name: Install npins - run: nix profile install 'nixpkgs#npins' - - name: Update npins sources - uses: getchoo/update-npins@v0 + run: nix-shell --run "npins update" + - name: Create PR + uses: peter-evans/create-pull-request@v7 with: token: "${{ secrets.DEPLOY_KEY }}" + branch: npins-update + commit-message: "npins: update sources" + title: "npins: update sources" diff --git a/README.md b/README.md index 3b19751c..d0a94395 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,3 @@ details as to what they are for. As an overview: - [`services/`](./services) contains our effort to make Fediverse applications work seemlessly together in our specific setting. - -- [`website/`](./website) contains the framework and the content of [the - Fediversity website](https://fediversity.eu/) diff --git a/default.nix b/default.nix index 4c71ec49..ea7b2557 100644 --- a/default.nix +++ b/default.nix @@ -9,6 +9,8 @@ let git-hooks gitignore ; + inherit (import sources.flake-inputs) import-flake; + inputs = (import-flake { src = ./.; }).inputs; inherit (pkgs) lib; pre-commit-check = (import "${git-hooks}/nix" { @@ -41,6 +43,23 @@ in shell = pkgs.mkShellNoCC { inherit (pre-commit-check) shellHook; buildInputs = pre-commit-check.enabledPackages; + packages = + let + test-loop = pkgs.writeShellApplication { + name = "test-loop"; + runtimeInputs = [ + pkgs.watchexec + pkgs.nix-unit + ]; + text = '' + watchexec -w ${builtins.toString ./.} -- nix-unit ${builtins.toString ./deployment/data-model-test.nix} "$@" + ''; + }; + in + [ + pkgs.nix-unit + test-loop + ]; }; tests = { @@ -50,6 +69,7 @@ in # re-export inputs so they can be overridden granularly # (they can't be accessed from the outside any other way) inherit + inputs sources system pkgs diff --git a/deployment/README.md b/deployment/README.md index f3e24276..f8eeabdc 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -3,6 +3,13 @@ This directory contains work to generate a full Fediversity deployment from a minimal configuration. This is different from [`../services/`](../services) that focuses on one machine, providing a polished and unified interface to different Fediverse services. +## Data model + +The core piece of the project is the [Fediversity data model](./data-model.nix), which describes all entities and their interactions. + +What can be done with it is exemplified in the [evaluation tests](./data-model-test.nix). +Run `test-loop` in the development environment when hacking on the data model or adding tests. + ## Checks There are three levels of deployment checks: `basic`, `cli`, `panel`. @@ -109,8 +116,8 @@ flowchart LR target_machines -->|get certs| acme ``` -### [WIP] Service deployment check from the panel +### Service deployment check from the FediPanel -This is a full deployment check running the panel on the deployer machine, deploying some services through the panel and checking that they are indeed on the target machines, then cleans them up and checks whether that works, too. +This is a full deployment check running the [FediPanel](../panel) on the deployer machine, deploying some services through it and checking that they are indeed on the target machines, then cleans them up and checks whether that works, too. -It builds upon the basic and CLI deployment checks. +It builds upon the basic and CLI deployment checks, the only difference being that `deployer` runs NixOps4 only indirectly via the panel, and the `client` node is the one that triggers the deployment via a browser, the way a human would. diff --git a/deployment/check/basic/nixosTest.nix b/deployment/check/basic/nixosTest.nix index 600baefb..fe9bda09 100644 --- a/deployment/check/basic/nixosTest.nix +++ b/deployment/check/basic/nixosTest.nix @@ -10,6 +10,12 @@ inputs.nixops4.packages.${pkgs.system}.default ]; + # FIXME: sad times + system.extraDependencies = with pkgs; [ + jq + jq.inputDerivation + ]; + system.extraDependenciesFromModule = { pkgs, ... }: { diff --git a/deployment/check/cli/deployer.pub b/deployment/check/cli/deployer.pub deleted file mode 100644 index 2303ffcb..00000000 --- a/deployment/check/cli/deployer.pub +++ /dev/null @@ -1 +0,0 @@ -## This is a placeholder file. It will be overwritten by the test. diff --git a/deployment/check/common/deployerNode.nix b/deployment/check/common/deployerNode.nix index 36f2897d..7236fc26 100644 --- a/deployment/check/common/deployerNode.nix +++ b/deployment/check/common/deployerNode.nix @@ -14,6 +14,8 @@ let types ; + sources = import ../../../npins; + in { imports = [ ./sharedOptions.nix ]; @@ -57,6 +59,8 @@ in "${inputs.nixops4-nixos}" "${inputs.nixpkgs}" + "${sources.flake-inputs}" + pkgs.stdenv pkgs.stdenvNoCC ] diff --git a/deployment/check/common/nixosTest.nix b/deployment/check/common/nixosTest.nix index ceec2726..aa91ae2e 100644 --- a/deployment/check/common/nixosTest.nix +++ b/deployment/check/common/nixosTest.nix @@ -119,7 +119,6 @@ in with subtest("Configure the deployer key"): deployer.succeed("""mkdir -p ~/.ssh && ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa""") deployer_key = deployer.succeed("cat ~/.ssh/id_rsa.pub").strip() - deployer.succeed(f"echo '{deployer_key}' > ${config.pathFromRoot}/deployer.pub") ${forConcat config.targetMachines (tm: '' ${tm}.succeed(f"mkdir -p /root/.ssh && echo '{deployer_key}' >> /root/.ssh/authorized_keys") '')} diff --git a/deployment/check/panel/flake-part.nix b/deployment/check/panel/flake-part.nix new file mode 100644 index 00000000..24a3d695 --- /dev/null +++ b/deployment/check/panel/flake-part.nix @@ -0,0 +1,91 @@ +{ + self, + inputs, + lib, + ... +}: + +let + inherit (builtins) + fromJSON + listToAttrs + ; + + targetMachines = [ + "garage" + "mastodon" + "peertube" + "pixelfed" + ]; + pathToRoot = /. + (builtins.unsafeDiscardStringContext self); + pathFromRoot = ./.; + enableAcme = true; + +in +{ + perSystem = + { pkgs, ... }: + { + checks.deployment-panel = pkgs.testers.runNixOSTest { + imports = [ + ../common/nixosTest.nix + ./nixosTest.nix + ]; + _module.args.inputs = inputs; + inherit + targetMachines + pathToRoot + pathFromRoot + enableAcme + ; + }; + }; + + nixops4Deployments = + let + makeTargetResource = nodeName: { + imports = [ ../common/targetResource.nix ]; + _module.args.inputs = inputs; + inherit + nodeName + pathToRoot + pathFromRoot + enableAcme + ; + }; + + ## The deployment function - what we are here to test! + ## + ## TODO: Modularise `deployment/default.nix` to get rid of the nested + ## function calls. + makeTestDeployment = + args: + (import ../..) + { + inherit lib; + inherit (inputs) nixops4 nixops4-nixos; + fediversity = import ../../../services/fediversity; + } + (listToAttrs ( + map (nodeName: { + name = "${nodeName}ConfigurationResource"; + value = makeTargetResource nodeName; + }) targetMachines + )) + args; + + in + { + check-deployment-panel = makeTestDeployment ( + fromJSON ( + let + env = builtins.getEnv "DEPLOYMENT"; + in + if env == "" then + throw "The DEPLOYMENT environment needs to be set. You do not want to use this deployment unless in the `deployment-panel` NixOS test." + else + env + ) + ); + }; +} diff --git a/deployment/check/panel/nixosTest.nix b/deployment/check/panel/nixosTest.nix new file mode 100644 index 00000000..39bd7930 --- /dev/null +++ b/deployment/check/panel/nixosTest.nix @@ -0,0 +1,362 @@ +{ + inputs, + lib, + hostPkgs, + config, + ... +}: + +let + inherit (lib) + getExe + ; + + ## Some places need a dummy file that will in fact never be used. We create + ## it here. + dummyFile = hostPkgs.writeText "dummy" "dummy"; + panelPort = 8000; + + panelUser = "test"; + panelEmail = "test@test.com"; + panelPassword = "ouiprdaaa43"; # panel's manager complains if too close to username or email + + fediUser = "test"; + fediEmail = "test@test.com"; + fediPassword = "testtest"; + fediName = "Testy McTestface"; + + toPythonBool = b: if b then "True" else "False"; + + interactWithPanel = + { + baseUri, + enableMastodon, + enablePeertube, + enablePixelfed, + }: + hostPkgs.writers.writePython3Bin "interact-with-panel" + { + libraries = with hostPkgs.python3Packages; [ selenium ]; + flakeIgnore = [ + "E302" # expected 2 blank lines, found 0 + "E303" # too many blank lines + "E305" # expected 2 blank lines after end of function or class + "E501" # line too long (> 79 characters) + "E731" # do not assign lambda expression, use a def + ]; + } + '' + from selenium import webdriver + from selenium.webdriver.common.by import By + from selenium.webdriver.firefox.options import Options + from selenium.webdriver.support.ui import WebDriverWait + + print("Create and configure driver...") + options = Options() + options.add_argument("--headless") + options.binary_location = "${getExe hostPkgs.firefox-unwrapped}" + service = webdriver.FirefoxService(executable_path="${getExe hostPkgs.geckodriver}") + driver = webdriver.Firefox(options=options, service=service) + driver.set_window_size(1280, 960) + driver.implicitly_wait(360) + driver.command_executor.set_timeout(3600) + + print("Open login page...") + driver.get("${baseUri}/login/") + print("Enter username...") + driver.find_element(By.XPATH, "//input[@name = 'username']").send_keys("${panelUser}") + print("Enter password...") + driver.find_element(By.XPATH, "//input[@name = 'password']").send_keys("${panelPassword}") + print("Click “Login” button...") + driver.find_element(By.XPATH, "//button[normalize-space() = 'Login']").click() + + print("Open configuration page...") + driver.get("${baseUri}/configuration/") + + # Helpers to actually set and not add or switch input values. + def input_set(elt, keys): + elt.clear() + elt.send_keys(keys) + def checkbox_set(elt, new_value): + if new_value != elt.is_selected(): + elt.click() + + print("Enable Fediversity...") + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'enable']"), True) + + print("Fill in initialUser info...") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.username']"), "${fediUser}") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.password']"), "${fediPassword}") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.email']"), "${fediEmail}") + input_set(driver.find_element(By.XPATH, "//input[@name = 'initialUser.displayName']"), "${fediName}") + + print("Enable services...") + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'mastodon.enable']"), ${toPythonBool enableMastodon}) + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'peertube.enable']"), ${toPythonBool enablePeertube}) + checkbox_set(driver.find_element(By.XPATH, "//input[@name = 'pixelfed.enable']"), ${toPythonBool enablePixelfed}) + + print("Start deployment...") + driver.find_element(By.XPATH, "//button[@id = 'deploy-button']").click() + + print("Wait for deployment status to show up...") + get_deployment_result = lambda d: d.find_element(By.XPATH, "//div[@id = 'deployment-result']//p") + WebDriverWait(driver, timeout=3660, poll_frequency=10).until(get_deployment_result) + deployment_result = get_deployment_result(driver).get_attribute('innerHTML') + + print("Quit...") + driver.quit() + + match deployment_result: + case 'Deployment Succeeded': + print("Deployment has succeeded; exiting normally") + exit(0) + case 'Deployment Failed': + print("Deployment has failed; exiting with return code `1`") + exit(1) + case _: + print(f"Unexpected deployment result: {deployment_result}; exiting with return code `2`") + exit(2) + ''; + +in + +{ + name = "deployment-panel"; + + ## The panel's module sets `nixpkgs.overlays` which clashes with + ## `pkgsReadOnly`. We disable it here. + node.pkgsReadOnly = false; + + nodes.deployer = + { pkgs, ... }: + { + imports = [ + (import ../../../panel { }).module + ]; + + ## FIXME: This should be in the common stuff. + security.acme = { + acceptTerms = true; + defaults.email = "test@test.com"; + defaults.server = "https://acme.test/dir"; + }; + security.pki.certificateFiles = [ + (import "${inputs.nixpkgs}/nixos/tests/common/acme/server/snakeoil-certs.nix").ca.cert + ]; + networking.extraHosts = "${config.acmeNodeIP} acme.test"; + + services.panel = { + enable = true; + production = true; + domain = "deployer"; + secrets = { + SECRET_KEY = dummyFile; + }; + port = panelPort; + nixops4Package = inputs.nixops4.packages.${pkgs.system}.default; + + deployment = { + flake = "/run/fedipanel/flake"; + name = "check-deployment-panel"; + }; + }; + + environment.systemPackages = [ pkgs.expect ]; + + ## FIXME: The following dependencies are necessary but I do not + ## understand why they are not covered by the fake node. + system.extraDependencies = with pkgs; [ + peertube + peertube.inputDerivation + gixy # a configuration checker for nginx + gixy.inputDerivation + ]; + + system.extraDependenciesFromModule = { + imports = [ ../../../services/fediversity ]; + fediversity = { + domain = "fediversity.net"; # would write `dummy` but that would not type + garage.enable = true; + mastodon = { + enable = true; + s3AccessKeyFile = dummyFile; + s3SecretKeyFile = dummyFile; + }; + peertube = { + enable = true; + secretsFile = dummyFile; + s3AccessKeyFile = dummyFile; + s3SecretKeyFile = dummyFile; + }; + pixelfed = { + enable = true; + s3AccessKeyFile = dummyFile; + s3SecretKeyFile = dummyFile; + }; + temp.cores = 1; + temp.initialUser = { + username = "dummy"; + displayName = "dummy"; + email = "dummy"; + passwordFile = dummyFile; + }; + }; + }; + }; + + nodes.client = + { pkgs, ... }: + { + environment.systemPackages = with pkgs; [ + httpie + dnsutils # for `dig` + openssl + cacert + wget + python3 + python3Packages.selenium + firefox-unwrapped + geckodriver + ]; + + security.pki.certificateFiles = [ + config.nodes.acme.test-support.acme.caCert + ]; + networking.extraHosts = "${config.acmeNodeIP} acme.test"; + }; + + ## NOTE: The target machines may need more RAM than the default to handle + ## being deployed to, otherwise we get something like: + ## + ## pixelfed # [ 616.785499 ] sshd-session[1167]: Conection closed by 2001:db8:1::2 port 45004 + ## deployer # error: writing to file: No space left on device + ## pixelfed # [ 616.788538 ] sshd-session[1151]: pam_unix(sshd:session): session closed for user port + ## pixelfed # [ 616.793929 ] systemd-logind[719]: Session 4 logged out. Waiting for processes to exit. + ## deployer # Error: Could not create resource + ## + ## These values have been trimmed down to the gigabyte. + nodes.mastodon.virtualisation.memorySize = 4 * 1024; + nodes.pixelfed.virtualisation.memorySize = 4 * 1024; + nodes.peertube.virtualisation.memorySize = 5 * 1024; + + ## FIXME: The test of presence of the services are very simple: we only + ## check that there is a systemd service of the expected name on the + ## machine. This proves at least that NixOps4 did something, and we cannot + ## really do more for now because the services aren't actually working + ## properly, in particular because of DNS issues. We should fix the services + ## and check that they are working properly. + + extraTestScript = '' + ## TODO: We want a nicer way to control where the FediPanel consumes its + ## flake, which can default to the store but could also be somewhere else if + ## someone wanted to change the code of the flake. + ## + with subtest("Give the panel access to the flake"): + deployer.succeed("mkdir /run/fedipanel /run/fedipanel/flake >&2") + deployer.succeed("cp -R . /run/fedipanel/flake >&2") + deployer.succeed("chown -R panel:panel /run/fedipanel >&2") + + ## TODO: I want a programmatic way to provide an SSH key to the panel (and + ## therefore NixOps4). This should happen either in the Python code, but + ## maybe it is fair that that one picks up on the user's key? It could + ## also be in the Nix packaging. + ## + with subtest("Set up the panel's SSH keys"): + deployer.succeed("mkdir /home/panel/.ssh >&2") + deployer.succeed("cp -R /root/.ssh/* /home/panel/.ssh >&2") + deployer.succeed("chown -R panel:panel /home/panel/.ssh >&2") + deployer.succeed("chmod 600 /home/panel/.ssh/* >&2") + + ## TODO: This is a hack to accept the root CA used by Pebble on the client + ## machine. Pebble randomizes everything, so the only way to get it is to + ## call the /roots/0 endpoint at runtime, leaving not much margin for a nice + ## Nixy way of adding the certificate. There is no way around it as this is + ## by design in Pebble, showing in fact that Pebble was not the appropriate + ## tool for our use and that nixpkgs does not in fact provide an easy way to + ## generate _usable_ certificates in NixOS tests. I suggest we merge this, + ## and track the task to set it up in a cleaner way. I would tackle this in + ## a subsequent PR, and hopefully even contribute this BetterWay(tm) to + ## nixpkgs. — Niols + ## + with subtest("Set up ACME root CA on client"): + client.succeed(""" + cd /etc/ssl/certs + curl -o pebble-root-ca.pem https://acme.test:15000/roots/0 + curl -o pebble-intermediate-ca.pem https://acme.test:15000/intermediates/0 + { cat ca-bundle.crt + cat pebble-root-ca.pem + cat pebble-intermediate-ca.pem + } > new-ca-bundle.crt + rm ca-bundle.crt ca-certificates.crt + mv new-ca-bundle.crt ca-bundle.crt + ln -s ca-bundle.crt ca-certificates.crt + """) + + ## TODO: I would hope for a more declarative way to add users. This should + ## be handled by the Nix packaging of the FediPanel. — Niols + ## + with subtest("Create panel user"): + deployer.succeed(""" + expect -c ' + spawn manage createsuperuser --username ${panelUser} --email ${panelEmail} + expect "Password: "; send "${panelPassword}\\n"; + expect "Password (again): "; send "${panelPassword}\\n" + interact + ' >&2 + """) + + with subtest("Check the status of the services - there should be none"): + garage.fail("systemctl status garage.service") + mastodon.fail("systemctl status mastodon-web.service") + peertube.fail("systemctl status peertube.service") + pixelfed.fail("systemctl status phpfpm-pixelfed.service") + + with subtest("Run deployment with no services enabled"): + client.succeed("${ + interactWithPanel { + baseUri = "https://deployer"; + enableMastodon = false; + enablePeertube = false; + enablePixelfed = false; + } + }/bin/interact-with-panel >&2") + + with subtest("Check the status of the services - there should still be none"): + garage.fail("systemctl status garage.service") + mastodon.fail("systemctl status mastodon-web.service") + peertube.fail("systemctl status peertube.service") + pixelfed.fail("systemctl status phpfpm-pixelfed.service") + + with subtest("Run deployment with Mastodon and Pixelfed enabled"): + client.succeed("${ + interactWithPanel { + baseUri = "https://deployer"; + enableMastodon = true; + enablePeertube = false; + enablePixelfed = true; + } + }/bin/interact-with-panel >&2") + + with subtest("Check the status of the services - expecting Garage, Mastodon and Pixelfed"): + garage.succeed("systemctl status garage.service") + mastodon.succeed("systemctl status mastodon-web.service") + peertube.fail("systemctl status peertube.service") + pixelfed.succeed("systemctl status phpfpm-pixelfed.service") + + with subtest("Run deployment with only Peertube enabled"): + client.succeed("${ + interactWithPanel { + baseUri = "https://deployer"; + enableMastodon = false; + enablePeertube = true; + enablePixelfed = false; + } + }/bin/interact-with-panel >&2") + + with subtest("Check the status of the services - expecting Garage and Peertube"): + garage.succeed("systemctl status garage.service") + mastodon.fail("systemctl status mastodon-web.service") + peertube.succeed("systemctl status peertube.service") + pixelfed.fail("systemctl status phpfpm-pixelfed.service") + ''; +} diff --git a/deployment/data-model-test.nix b/deployment/data-model-test.nix new file mode 100644 index 00000000..34f9b381 --- /dev/null +++ b/deployment/data-model-test.nix @@ -0,0 +1,205 @@ +let + inherit (import ../default.nix { }) pkgs inputs; + inherit (pkgs) lib; + inherit (lib) mkOption types; + eval = + module: + (lib.evalModules { + specialArgs = { + inherit inputs; + }; + modules = [ + module + ./data-model.nix + ]; + }).config; + nixops4Deployment = inputs.nixops4.modules.nixops4Deployment.default; +in +{ + test-eval = { + expr = + let + fediversity = eval ( + { config, ... }: + { + config = { + resources.nixos-configuration = { + description = "An entire NixOS configuration"; + request = + { ... }: + { + _class = "fediversity-resource-request"; + options.config = mkOption { + description = "Any options from NixOS"; + }; + }; + + policy = + { config, ... }: + { + _class = "fediversity-resource-policy"; + + options = { + extra-config = mkOption { + description = "Any options from NixOS"; + }; + }; + config.resource-type = types.raw; # TODO: what's the type of a NixOS configuration? + config.apply = requests: lib.mkMerge (requests ++ [ config.extra-config ]); + }; + }; + resources.login-shell = { + description = "The operator needs to be able to log into the shell"; + request = + { ... }: + { + _class = "fediversity-resource-request"; + options = { + wheel = mkOption { + description = "Whether the login user needs root permissions"; + type = types.bool; + default = false; + }; + packages = mkOption { + description = "Packages that need to be available in the user environment"; + type = with types; attrsOf package; + }; + }; + }; + policy = + { config, ... }: + { + _class = "fediversity-resource-policy"; + options = { + username = mkOption { + description = "Username for the operator"; + type = types.str; # TODO: use the proper constraints from NixOS + }; + wheel = mkOption { + description = "Whether to allow login with root permissions"; + type = types.bool; + default = false; + }; + }; + config.resource-type = types.raw; # TODO: splice out the user type from NixOS + config.apply = + requests: + let + # Filter out requests that need wheel if policy doesn't allow it + validRequests = lib.filterAttrs ( + _name: req: !req.login-shell.wheel || config.wheel + ) requests.resources; + in + lib.optionalAttrs (validRequests != { }) { + ${config.username} = { + isNormalUser = true; + packages = + with lib; + attrValues (concatMapAttrs (_name: request: request.login-shell.packages) validRequests); + extraGroups = lib.optional config.wheel "wheel"; + }; + }; + }; + }; + applications.hello = + { ... }: + { + description = ''Command-line tool that will print "Hello, world!" on the terminal''; + module = + { ... }: + { + options.enable = lib.mkEnableOption "Hello in the shell"; + }; + implementation = cfg: { + input = cfg; + output = lib.optionalAttrs cfg.enable { + resources.hello.login-shell.packages.hello = pkgs.hello; + }; + }; + }; + environments.single-nixos-vm = + { config, ... }: + { + resources.operator-environment.login-shell.username = "operator"; + implementation = requests: { + input = requests; + output = + { providers, ... }: + { + providers = { + inherit (inputs.nixops4.modules.nixops4Provider) local; + }; + resources.the-machine = { + type = providers.local.exec; + imports = [ + inputs.nixops4-nixos.modules.nixops4Resource.nixos + ]; + nixos.module = + { ... }: + { + users.users = config.resources.shell.login-shell.apply ( + lib.filterAttrs (_name: value: value ? login-shell) requests + ); + }; + }; + }; + }; + }; + }; + options = { + example-configuration = mkOption { + type = config.configuration; + readOnly = true; + default = { + enable = true; + applications.hello.enable = true; + }; + }; + example-deployment = mkOption { + type = types.submodule nixops4Deployment; + readOnly = true; + default = config.environments.single-nixos-vm.deployment config.example-configuration; + }; + }; + } + ); + in + rec { + number-of-resources = with lib; length (attrNames fediversity.resources); + inherit (fediversity) example-configuration; + hello-package-exists = + (fediversity.applications.hello.resources example-configuration.applications.hello) + .resources.hello.login-shell.packages ? hello; + wheel-required = + (fediversity.applications.hello.resources example-configuration.applications.hello) + .resources.hello.login-shell.wheel; + wheel-allowed = + fediversity.environments.single-nixos-vm.resources.operator-environment.login-shell.wheel; + operator-shell = + let + resources = fediversity.applications.hello.resources example-configuration.applications.hello; + users = fediversity.environments.single-nixos-vm.resources.operator-environment.login-shell.apply resources; + in + { + inherit (users.operator) isNormalUser; + packages = with lib; map (p: "${p.pname}") users.operator.packages; + extraGroups = users.operator.extraGroups; + }; + }; + expected = { + number-of-resources = 2; + example-configuration = { + enable = true; + applications.hello.enable = true; + }; + hello-package-exists = true; + wheel-required = false; + wheel-allowed = false; + operator-shell = { + isNormalUser = true; + packages = [ "hello" ]; + extraGroups = [ ]; + }; + }; + }; +} diff --git a/deployment/data-model.nix b/deployment/data-model.nix new file mode 100644 index 00000000..ff157149 --- /dev/null +++ b/deployment/data-model.nix @@ -0,0 +1,200 @@ +{ + lib, + config, + inputs, + ... +}: +let + inherit (lib) mkOption types; + inherit (lib.types) + attrsOf + attrTag + deferredModuleWith + submoduleWith + submodule + optionType + functionTo + ; + + functionType = import ./function.nix; + application-resources = submodule { + options.resources = mkOption { + # TODO: maybe transpose, and group the resources by type instead + type = attrsOf ( + attrTag ( + lib.mapAttrs (_name: resource: mkOption { type = submodule resource.request; }) config.resources + ) + ); + }; + }; + nixops4Deployment = types.deferredModuleWith { + staticModules = [ + inputs.nixops4.modules.nixops4Deployment.default + + { + _module.args = { + resourceProviderSystem = builtins.currentSystem; + resources = { }; + }; + } + ]; + }; +in +{ + options = { + resources = mkOption { + description = "Collection of deployment resources that can be required by applications and policed by hosting providers"; + type = attrsOf ( + submodule ( + { ... }: + { + _class = "fediversity-resource"; + options = { + description = mkOption { + description = "Description of the resource to help application module authors and hosting providers to work with it"; + type = types.str; + }; + request = mkOption { + description = "Options for declaring resource requirements by an application, a description of how the resource is consumed or accessed"; + type = deferredModuleWith { staticModules = [ { _class = "fediversity-resource-request"; } ]; }; + }; + policy = mkOption { + description = "Options for configuring the resource policy for the hosting provider, a description of how the resource is made available"; + type = deferredModuleWith { + staticModules = [ + (policy: { + _class = "fediversity-resource-policy"; + # TODO(@fricklerhandwerk): not sure it can be made + # sensible syntactically, but essentially we want to + # ensure that `apply` is defined, but since its output + # depends on the specific policy we also need to + # determine that somehow. + # hopefully this also helps with correct composition down the line. + options.resource-type = mkOption { + description = "The type of resource this policy configures"; + type = types.optionType; + }; + # TODO(@fricklerhandwerk): do we need a function type here as well, or is it in the way? + options.apply = mkOption { + description = "Apply the policy to a request"; + type = with types; functionTo policy.config.resource-type; + }; + }) + ]; + }; + }; + }; + } + ) + ); + }; + applications = mkOption { + description = "Collection of Fediversity applications"; + type = attrsOf ( + submodule (application: { + _class = "fediversity-application"; + options = { + description = mkOption { + description = "Description to be shown in the application overview"; + type = types.str; + }; + module = mkOption { + description = "Operator-facing configuration options for the application"; + type = deferredModuleWith { staticModules = [ { _class = "fediversity-application-config"; } ]; }; + }; + implementation = mkOption { + description = "Mapping of application configuration to deployment resources, a description of what an application needs to run"; + type = application.config.config-mapping.function-type; + }; + resources = mkOption { + description = "Compute resources required by an application"; + type = functionTo application.config.config-mapping.output-type; + readOnly = true; + default = input: (application.config.implementation input).output; + }; + config-mapping = mkOption { + description = "Function type for the mapping from application configuration to required resources"; + type = submodule functionType; + readOnly = true; + default = { + input-type = submodule application.config.module; + output-type = application-resources; + }; + }; + }; + }) + ); + }; + environments = mkOption { + description = "Run-time environments for Fediversity applications to be deployed to"; + type = attrsOf ( + submodule (environment: { + _class = "fediversity-environment"; + options = { + resources = mkOption { + description = '' + Resources made available by the hosting provider, and their policies. + + Setting this is optional, but provides a place to declare that information for programmatic use in the resource mapping. + ''; + # TODO: maybe transpose, and group the resources by type instead + type = attrsOf ( + attrTag ( + lib.mapAttrs (_name: resource: mkOption { type = submodule resource.policy; }) config.resources + ) + ); + }; + implementation = mkOption { + description = "Mapping of resources required by applications to available resources; the result can be deployed"; + type = environment.config.resource-mapping.function-type; + }; + resource-mapping = mkOption { + description = "Function type for the mapping from resources to a (NixOps4) deployment"; + type = submodule functionType; + readOnly = true; + default = { + input-type = application-resources; + output-type = types.raw; + }; + }; + deployment = mkOption { + description = "Generate a deployment from a configuration"; + type = functionTo environment.config.resource-mapping.output-type; + readOnly = true; + default = + cfg: + # TODO: check cfg.enable.true + let + required-resources = lib.mapAttrs ( + name: application-settings: config.applications.${name}.resources application-settings + ) cfg.applications; + in + (environment.config.implementation required-resources).output; + + }; + }; + }) + ); + }; + configuration = mkOption { + description = "Configuration type declaring options to be set by operators"; + type = optionType; + readOnly = true; + default = submodule { + options = { + enable = lib.mkEnableOption { + description = "your Fediversity configuration"; + }; + applications = lib.mapAttrs ( + _name: application: + mkOption { + description = application.description; + type = submodule application.module; + default = { }; + } + ) config.applications; + }; + }; + }; + }; +} diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix index 5e822688..4f31f2eb 100644 --- a/deployment/flake-part.nix +++ b/deployment/flake-part.nix @@ -2,5 +2,6 @@ imports = [ ./check/basic/flake-part.nix ./check/cli/flake-part.nix + ./check/panel/flake-part.nix ]; } diff --git a/deployment/function.nix b/deployment/function.nix new file mode 100644 index 00000000..5067505c --- /dev/null +++ b/deployment/function.nix @@ -0,0 +1,38 @@ +/** + Modular function type +*/ +{ config, lib, ... }: +let + inherit (lib) mkOption types; + inherit (types) + submodule + functionTo + optionType + ; +in +{ + options = { + input-type = mkOption { + type = optionType; + }; + output-type = mkOption { + type = optionType; + }; + function-type = mkOption { + type = optionType; + readOnly = true; + default = functionTo ( + submodule (function: { + options = { + input = mkOption { + type = config.input-type; + }; + output = mkOption { + type = config.output-type; + }; + }; + }) + ); + }; + }; +} diff --git a/flake.lock b/flake.lock index 4eff9508..baf0a2bc 100644 --- a/flake.lock +++ b/flake.lock @@ -596,22 +596,6 @@ "type": "github" } }, - "nixpkgs_4": { - "locked": { - "lastModified": 1740463929, - "narHash": "sha256-4Xhu/3aUdCKeLfdteEHMegx5ooKQvwPHNkOgNCXQrvc=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "5d7db4668d7a0c6cc5fc8cf6ef33b008b2b1ed8b", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixos-24.11", - "repo": "nixpkgs", - "type": "github" - } - }, "parts": { "inputs": { "nixpkgs-lib": [ @@ -686,8 +670,7 @@ "nixops4-nixos", "nixops4" ], - "nixops4-nixos": "nixops4-nixos", - "nixpkgs": "nixpkgs_4" + "nixops4-nixos": "nixops4-nixos" } }, "rust-overlay": { diff --git a/flake.nix b/flake.nix index 6dd3d3df..e5b6a103 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,5 @@ { inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; # consumed by flake-parts flake-parts.url = "github:hercules-ci/flake-parts"; git-hooks.url = "github:cachix/git-hooks.nix"; nixops4.follows = "nixops4-nixos/nixops4"; @@ -8,65 +7,88 @@ }; outputs = - inputs@{ flake-parts, ... }: + inputs@{ self, flake-parts, ... }: let sources = import ./npins; + inherit (import sources.flake-inputs) import-flake; inherit (sources) git-hooks agenix; + # XXX(@fricklerhandwerk): this atrocity is required to splice in a foreign Nixpkgs via flake-parts + # XXX - this is just importing a flake + nixpkgs = import-flake { src = sources.nixpkgs; }; + # XXX - this overrides the inputs attached to `self` + inputs' = self.inputs // { + nixpkgs = nixpkgs; + }; + self' = self // { + inputs = inputs'; + }; in - flake-parts.lib.mkFlake { inherit inputs; } { - systems = [ - "x86_64-linux" - "aarch64-linux" - "x86_64-darwin" - "aarch64-darwin" - ]; - - imports = [ - (import "${git-hooks}/flake-module.nix") - inputs.nixops4.modules.flake.default - - ./deployment/flake-part.nix - ./infra/flake-part.nix - ]; - - perSystem = - { - pkgs, - lib, - inputs', - ... - }: - { - formatter = pkgs.nixfmt-rfc-style; - - pre-commit.settings.hooks = - let - ## Add a directory here if pre-commit hooks shouldn't apply to it. - optout = [ "npins" ]; - excludes = map (dir: "^${dir}/") optout; - addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); - in - addExcludes { - nixfmt-rfc-style.enable = true; - deadnix.enable = true; - trim-trailing-whitespace.enable = true; - shellcheck.enable = true; - }; - - devShells.default = pkgs.mkShell { - packages = [ - pkgs.npins - pkgs.nil - (pkgs.callPackage "${agenix}/pkgs/agenix.nix" { }) - pkgs.openssh - pkgs.httpie - pkgs.jq - # exposing this env var as a hack to pass info in from form - (inputs'.nixops4.packages.default.overrideAttrs { - impureEnvVars = [ "DEPLOYMENT" ]; - }) - ]; - }; + # XXX - finally we override the overall set of `inputs` -- we need both: + # `flake-parts obtains `nixpkgs` from `self.inputs` and not from `inputs`. + flake-parts.lib.mkFlake + { + inputs = inputs // { + inherit nixpkgs; }; - }; + self = self'; + } + ( + { inputs, ... }: + { + systems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + imports = [ + (import "${git-hooks}/flake-module.nix") + inputs.nixops4.modules.flake.default + + ./deployment/flake-part.nix + ./infra/flake-part.nix + ]; + + perSystem = + { + pkgs, + lib, + inputs', + ... + }: + { + formatter = pkgs.nixfmt-rfc-style; + + pre-commit.settings.hooks = + let + ## Add a directory here if pre-commit hooks shouldn't apply to it. + optout = [ "npins" ]; + excludes = map (dir: "^${dir}/") optout; + addExcludes = lib.mapAttrs (_: c: c // { inherit excludes; }); + in + addExcludes { + nixfmt-rfc-style.enable = true; + deadnix.enable = true; + trim-trailing-whitespace.enable = true; + shellcheck.enable = true; + }; + + devShells.default = pkgs.mkShell { + packages = [ + pkgs.npins + pkgs.nil + (pkgs.callPackage "${agenix}/pkgs/agenix.nix" { }) + pkgs.openssh + pkgs.httpie + pkgs.jq + # exposing this env var as a hack to pass info in from form + (inputs'.nixops4.packages.default.overrideAttrs { + impureEnvVars = [ "DEPLOYMENT" ]; + }) + ]; + }; + }; + } + ); } diff --git a/infra/README.md b/infra/README.md index 133f6a32..422680fd 100644 --- a/infra/README.md +++ b/infra/README.md @@ -14,7 +14,7 @@ everything will become much cleaner. above 100. For instance, `fedi117`. 2. Add a basic configuration for the machine. These typically go in - `infra/machines//default.nix`. You can look at other `fediXXX` VMs to + `machines/dev//default.nix`. You can look at other `fediXXX` VMs to find inspiration. You probably do not need a `nixos.module` option at this point. @@ -48,7 +48,7 @@ everything will become much cleaner. 7. Regenerate the list of machines: ``` - sh infra/machines.md.sh + sh machines/machines.md.sh ``` Commit it with the machine's configuration, public key, etc. diff --git a/infra/common/resource.nix b/infra/common/resource.nix index 7e86467c..ebaea6c6 100644 --- a/infra/common/resource.nix +++ b/infra/common/resource.nix @@ -1,4 +1,5 @@ { + inputs, lib, config, ... @@ -9,7 +10,7 @@ let inherit (lib.attrsets) concatMapAttrs optionalAttrs; inherit (lib.strings) removeSuffix; sources = import ../../npins; - inherit (sources) nixpkgs agenix disko; + inherit (sources) agenix disko; secretsPrefix = ../../secrets; secrets = import (secretsPrefix + "/secrets.nix"); @@ -26,7 +27,7 @@ in hostPublicKey = config.fediversityVm.hostPublicKey; }; - inherit nixpkgs; + inherit (inputs) nixpkgs; ## The configuration of the machine. We strive to keep in this file only the ## options that really need to be injected from the resource. Everything else diff --git a/infra/flake-part.nix b/infra/flake-part.nix index af0fe51d..068bf6a6 100644 --- a/infra/flake-part.nix +++ b/infra/flake-part.nix @@ -21,6 +21,9 @@ let makeResourceModule = { vmName, isTestVm }: { + # TODO(@fricklerhandwerk): this is terrible but IMO we should just ditch flake-parts and have our own data model for how the project is organised internally + _module.args = { inherit inputs; }; + imports = [ ./common/resource.nix @@ -28,7 +31,7 @@ let ++ ( if isTestVm then [ - ./test-machines/${vmName} + ../machines/operator/${vmName} { nixos.module.users.users.root.openssh.authorizedKeys.keys = [ # allow our panel vm access to the test machines @@ -38,7 +41,7 @@ let ] else [ - ./machines/${vmName} + ../machines/dev/${vmName} ] ); fediversityVm.name = vmName; @@ -147,8 +150,8 @@ let listSubdirectories = path: attrNames (filterAttrs (_: type: type == "directory") (readDir path)); - machines = listSubdirectories ./machines; - testMachines = listSubdirectories ./test-machines; + machines = listSubdirectories ../machines/dev; + testMachines = listSubdirectories ../machines/operator; in { diff --git a/machines/README.md b/machines/README.md new file mode 100644 index 00000000..d51df126 --- /dev/null +++ b/machines/README.md @@ -0,0 +1,4 @@ +# Machines + +This directory contains the definition of [the VMs](machines.md) that host our +infrastructure. diff --git a/infra/machines/fedi200/default.nix b/machines/dev/fedi200/default.nix similarity index 100% rename from infra/machines/fedi200/default.nix rename to machines/dev/fedi200/default.nix diff --git a/infra/machines/fedi201/default.nix b/machines/dev/fedi201/default.nix similarity index 100% rename from infra/machines/fedi201/default.nix rename to machines/dev/fedi201/default.nix diff --git a/infra/machines/fedi201/fedipanel.nix b/machines/dev/fedi201/fedipanel.nix similarity index 100% rename from infra/machines/fedi201/fedipanel.nix rename to machines/dev/fedi201/fedipanel.nix diff --git a/infra/machines/vm02116/default.nix b/machines/dev/vm02116/default.nix similarity index 100% rename from infra/machines/vm02116/default.nix rename to machines/dev/vm02116/default.nix diff --git a/infra/machines/vm02116/forgejo.nix b/machines/dev/vm02116/forgejo.nix similarity index 100% rename from infra/machines/vm02116/forgejo.nix rename to machines/dev/vm02116/forgejo.nix diff --git a/infra/machines/vm02187/default.nix b/machines/dev/vm02187/default.nix similarity index 100% rename from infra/machines/vm02187/default.nix rename to machines/dev/vm02187/default.nix diff --git a/infra/machines/vm02187/wiki.nix b/machines/dev/vm02187/wiki.nix similarity index 100% rename from infra/machines/vm02187/wiki.nix rename to machines/dev/vm02187/wiki.nix diff --git a/infra/machines.md b/machines/machines.md similarity index 100% rename from infra/machines.md rename to machines/machines.md diff --git a/infra/machines.md.sh b/machines/machines.md.sh similarity index 92% rename from infra/machines.md.sh rename to machines/machines.md.sh index ea1b0208..d523e127 100644 --- a/infra/machines.md.sh +++ b/machines/machines.md.sh @@ -20,7 +20,7 @@ vmOptions=$( cd .. nix eval \ --impure --raw --expr " - builtins.toJSON (builtins.getFlake (builtins.toString ./.)).vmOptions + builtins.toJSON (builtins.getFlake (builtins.toString ../.)).vmOptions " \ --log-format raw --quiet ) diff --git a/infra/test-machines/test01/default.nix b/machines/operator/test01/default.nix similarity index 100% rename from infra/test-machines/test01/default.nix rename to machines/operator/test01/default.nix diff --git a/infra/test-machines/test01/ssh_host_ed25519_key b/machines/operator/test01/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test01/ssh_host_ed25519_key rename to machines/operator/test01/ssh_host_ed25519_key diff --git a/infra/test-machines/test01/ssh_host_ed25519_key.pub b/machines/operator/test01/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test01/ssh_host_ed25519_key.pub rename to machines/operator/test01/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test02/default.nix b/machines/operator/test02/default.nix similarity index 100% rename from infra/test-machines/test02/default.nix rename to machines/operator/test02/default.nix diff --git a/infra/test-machines/test02/ssh_host_ed25519_key b/machines/operator/test02/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test02/ssh_host_ed25519_key rename to machines/operator/test02/ssh_host_ed25519_key diff --git a/infra/test-machines/test02/ssh_host_ed25519_key.pub b/machines/operator/test02/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test02/ssh_host_ed25519_key.pub rename to machines/operator/test02/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test03/default.nix b/machines/operator/test03/default.nix similarity index 100% rename from infra/test-machines/test03/default.nix rename to machines/operator/test03/default.nix diff --git a/infra/test-machines/test03/ssh_host_ed25519_key b/machines/operator/test03/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test03/ssh_host_ed25519_key rename to machines/operator/test03/ssh_host_ed25519_key diff --git a/infra/test-machines/test03/ssh_host_ed25519_key.pub b/machines/operator/test03/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test03/ssh_host_ed25519_key.pub rename to machines/operator/test03/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test04/default.nix b/machines/operator/test04/default.nix similarity index 100% rename from infra/test-machines/test04/default.nix rename to machines/operator/test04/default.nix diff --git a/infra/test-machines/test04/ssh_host_ed25519_key b/machines/operator/test04/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test04/ssh_host_ed25519_key rename to machines/operator/test04/ssh_host_ed25519_key diff --git a/infra/test-machines/test04/ssh_host_ed25519_key.pub b/machines/operator/test04/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test04/ssh_host_ed25519_key.pub rename to machines/operator/test04/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test05/default.nix b/machines/operator/test05/default.nix similarity index 100% rename from infra/test-machines/test05/default.nix rename to machines/operator/test05/default.nix diff --git a/infra/test-machines/test05/ssh_host_ed25519_key b/machines/operator/test05/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test05/ssh_host_ed25519_key rename to machines/operator/test05/ssh_host_ed25519_key diff --git a/infra/test-machines/test05/ssh_host_ed25519_key.pub b/machines/operator/test05/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test05/ssh_host_ed25519_key.pub rename to machines/operator/test05/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test06/default.nix b/machines/operator/test06/default.nix similarity index 100% rename from infra/test-machines/test06/default.nix rename to machines/operator/test06/default.nix diff --git a/infra/test-machines/test06/ssh_host_ed25519_key b/machines/operator/test06/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test06/ssh_host_ed25519_key rename to machines/operator/test06/ssh_host_ed25519_key diff --git a/infra/test-machines/test06/ssh_host_ed25519_key.pub b/machines/operator/test06/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test06/ssh_host_ed25519_key.pub rename to machines/operator/test06/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test11/default.nix b/machines/operator/test11/default.nix similarity index 100% rename from infra/test-machines/test11/default.nix rename to machines/operator/test11/default.nix diff --git a/infra/test-machines/test11/ssh_host_ed25519_key b/machines/operator/test11/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test11/ssh_host_ed25519_key rename to machines/operator/test11/ssh_host_ed25519_key diff --git a/infra/test-machines/test11/ssh_host_ed25519_key.pub b/machines/operator/test11/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test11/ssh_host_ed25519_key.pub rename to machines/operator/test11/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test12/default.nix b/machines/operator/test12/default.nix similarity index 100% rename from infra/test-machines/test12/default.nix rename to machines/operator/test12/default.nix diff --git a/infra/test-machines/test12/ssh_host_ed25519_key b/machines/operator/test12/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test12/ssh_host_ed25519_key rename to machines/operator/test12/ssh_host_ed25519_key diff --git a/infra/test-machines/test12/ssh_host_ed25519_key.pub b/machines/operator/test12/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test12/ssh_host_ed25519_key.pub rename to machines/operator/test12/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test13/default.nix b/machines/operator/test13/default.nix similarity index 100% rename from infra/test-machines/test13/default.nix rename to machines/operator/test13/default.nix diff --git a/infra/test-machines/test13/ssh_host_ed25519_key b/machines/operator/test13/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test13/ssh_host_ed25519_key rename to machines/operator/test13/ssh_host_ed25519_key diff --git a/infra/test-machines/test13/ssh_host_ed25519_key.pub b/machines/operator/test13/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test13/ssh_host_ed25519_key.pub rename to machines/operator/test13/ssh_host_ed25519_key.pub diff --git a/infra/test-machines/test14/default.nix b/machines/operator/test14/default.nix similarity index 100% rename from infra/test-machines/test14/default.nix rename to machines/operator/test14/default.nix diff --git a/infra/test-machines/test14/ssh_host_ed25519_key b/machines/operator/test14/ssh_host_ed25519_key similarity index 100% rename from infra/test-machines/test14/ssh_host_ed25519_key rename to machines/operator/test14/ssh_host_ed25519_key diff --git a/infra/test-machines/test14/ssh_host_ed25519_key.pub b/machines/operator/test14/ssh_host_ed25519_key.pub similarity index 100% rename from infra/test-machines/test14/ssh_host_ed25519_key.pub rename to machines/operator/test14/ssh_host_ed25519_key.pub diff --git a/npins/sources.json b/npins/sources.json index 4971590b..a96ffcb2 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -25,6 +25,38 @@ "url": null, "hash": "1w2gsy6qwxa5abkv8clb435237iifndcxq0s79wihqw11a5yb938" }, + "disko": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "nix-community", + "repo": "disko" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "v1.12.0", + "revision": "7121f74b976481bc36877abaf52adab2a178fcbe", + "url": "https://api.github.com/repos/nix-community/disko/tarball/v1.12.0", + "hash": "0wbx518d2x54yn4xh98cgm65wvj0gpy6nia6ra7ns4j63hx14fkq" + }, + "flake-inputs": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "fricklerhandwerk", + "repo": "flake-inputs" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "4.1", + "revision": "ad02792f7543754569fe2fd3d5787ee00ef40be2", + "url": "https://api.github.com/repos/fricklerhandwerk/flake-inputs/tarball/4.1", + "hash": "1j57avx2mqjnhrsgq3xl7ih8v7bdhz1kj3min6364f486ys048bm" + }, "flake-parts": { "type": "Git", "repository": { diff --git a/panel/default.nix b/panel/default.nix index a9c20f84..c6749611 100644 --- a/panel/default.nix +++ b/panel/default.nix @@ -20,8 +20,15 @@ in packages = [ pkgs.npins manage + + # NixOps4 and its dependencies + # FIXME: grab NixOps4 and add it here + pkgs.nix + pkgs.openssh ]; - env = import ./env.nix { inherit lib pkgs; } // { + env = { + DEPLOYMENT_FLAKE = ../.; + DEPLOYMENT_NAME = "test"; NPINS_DIRECTORY = toString ../npins; CREDENTIALS_DIRECTORY = toString ./.credentials; DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3"; diff --git a/panel/env.nix b/panel/env.nix deleted file mode 100644 index 07ce4193..00000000 --- a/panel/env.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - lib, - pkgs, - ... -}: -let - inherit (builtins) toString; -in -{ - REPO_DIR = toString ../.; - # explicitly use nix, as e.g. lix does not have configurable-impure-env - BIN_PATH = lib.makeBinPath [ - # explicitly use nix, as e.g. lix does not have configurable-impure-env - pkgs.nix - # nixops error maybe due to our flake git hook: executing 'git': No such file or directory - pkgs.git - ]; -} diff --git a/panel/nix/configuration.nix b/panel/nix/configuration.nix index b0bcd5c4..ccc6506a 100644 --- a/panel/nix/configuration.nix +++ b/panel/nix/configuration.nix @@ -23,7 +23,9 @@ let cfg = config.services.${name}; package = pkgs.callPackage ./package.nix { }; - environment = import ../env.nix { inherit lib pkgs; } // { + environment = { + DEPLOYMENT_FLAKE = cfg.deployment.flake; + DEPLOYMENT_NAME = cfg.deployment.name; DATABASE_URL = "sqlite:////var/lib/${name}/db.sqlite3"; USER_SETTINGS_FILE = pkgs.concatText "configuration.py" [ ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings) @@ -133,6 +135,34 @@ in type = types.attrsOf types.path; default = { }; }; + nixops4Package = mkOption { + type = types.package; + description = '' + A package providing NixOps4. + + TODO: This should not be at the level of the NixOS module, but instead + at the level of the panel's package. Until one finds a way to grab + NixOps4 from the package's npins-based code, we will have to do with + this workaround. + ''; + }; + + deployment = { + flake = mkOption { + type = types.path; + default = ../..; + description = '' + The path to the flake containing the deployment. This is used to run the deployment button. + ''; + }; + name = mkOption { + type = types.str; + default = "test"; + description = '' + The name of the deployment within the flake. + ''; + }; + }; }; config = mkIf cfg.enable { @@ -170,6 +200,8 @@ in }; users.users.${name} = { + # TODO[Niols]: change to system user or document why we specifically + # need a normal user. isNormalUser = true; }; @@ -181,6 +213,11 @@ in path = [ python-environment manage-service + + ## NixOps4 and its dependencies + cfg.nixops4Package + pkgs.nix + pkgs.openssh ]; preStart = '' # Auto-migrate on first run or if the package has changed diff --git a/panel/nix/tests.nix b/panel/nix/tests.nix index 11009213..e76eaed0 100644 --- a/panel/nix/tests.nix +++ b/panel/nix/tests.nix @@ -13,6 +13,7 @@ let secrets = { SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret"; }; + nixops4Package = pkgs.hello; # FIXME: actually pass NixOps4 }; virtualisation = { diff --git a/panel/src/panel/settings.py b/panel/src/panel/settings.py index bbfa753a..5a555495 100644 --- a/panel/src/panel/settings.py +++ b/panel/src/panel/settings.py @@ -42,6 +42,7 @@ def get_secret(name: str, encoding: str = "utf-8") -> str: return secret # SECURITY WARNING: keep the secret key used in production secret! +# This is used nowhere but is required by Django. SECRET_KEY = get_secret("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! @@ -240,8 +241,7 @@ if user_settings_file is not None: # The correct thing to do here would be using a helper function such as with `get_secret()` that will catch the exception and explain what's wrong and where to put the right values. # Replacing the `USER_SETTINGS_FILE` mechanism following the comment there would probably be a good thing. -# PATH to expose to launch button -bin_path=env['BIN_PATH'] -# path of the root flake to trigger nixops from, see #94. -# to deploy this should be specified, for dev just use a relative path. -repo_dir = env["REPO_DIR"] +# Path of the root flake to trigger nixops from, see #94, and name of the +# deployment. +deployment_flake = env["DEPLOYMENT_FLAKE"] +deployment_name = env["DEPLOYMENT_NAME"] diff --git a/panel/src/panel/views.py b/panel/src/panel/views.py index 9b1e902e..2f603002 100644 --- a/panel/src/panel/views.py +++ b/panel/src/panel/views.py @@ -89,25 +89,20 @@ class DeploymentStatus(ConfigurationForm): def deployment(self, config: BaseModel): env = { - "PATH": settings.bin_path, + "PATH": os.environ.get("PATH"), # pass in form info to our deployment "DEPLOYMENT": config.json() } cmd = [ - "nix", - "develop", - "--extra-experimental-features", - "configurable-impure-env", - "--command", "nixops4", "apply", - "test", + settings.deployment_name, "--show-trace", "--no-interactive", ] deployment_result = subprocess.run( cmd, - cwd = settings.repo_dir, + cwd = settings.deployment_flake, env = env, stderr = subprocess.STDOUT, ) diff --git a/services/default.nix b/services/default.nix index a1c868ad..3117b861 100644 --- a/services/default.nix +++ b/services/default.nix @@ -6,8 +6,8 @@ }: { tests = { - mastodon = import ./tests/mastodon.nix { inherit pkgs; }; - pixelfed-garage = import ./tests/pixelfed-garage.nix { inherit pkgs; }; - peertube = import ./tests/peertube.nix { inherit pkgs; }; + mastodon = pkgs.nixosTest ./tests/mastodon.nix; + pixelfed-garage = pkgs.nixosTest ./tests/pixelfed-garage.nix; + peertube = pkgs.nixosTest ./tests/peertube.nix; }; } diff --git a/services/fediversity/pixelfed/default.nix b/services/fediversity/pixelfed/default.nix index d6328e3b..9080e6ba 100644 --- a/services/fediversity/pixelfed/default.nix +++ b/services/fediversity/pixelfed/default.nix @@ -56,12 +56,6 @@ in ) (mkIf config.fediversity.pixelfed.enable { - ## NOTE: Pixelfed as packaged in nixpkgs has a permission issue that prevents Nginx - ## from being able to serving the images. We fix it here, but this should be - ## upstreamed. See https://github.com/NixOS/nixpkgs/issues/235147 - services.pixelfed.package = pkgs.pixelfed.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ ./group-permissions.patch ]; - }); users.users.nginx.extraGroups = [ "pixelfed" ]; services.pixelfed = { diff --git a/services/fediversity/pixelfed/group-permissions.patch b/services/fediversity/pixelfed/group-permissions.patch deleted file mode 100644 index d7dd442d..00000000 --- a/services/fediversity/pixelfed/group-permissions.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/config/filesystems.php b/config/filesystems.php -index 00254e93..fc1a58f3 100644 ---- a/config/filesystems.php -+++ b/config/filesystems.php -@@ -49,11 +49,11 @@ return [ - 'permissions' => [ - 'file' => [ - 'public' => 0644, -- 'private' => 0600, -+ 'private' => 0640, - ], - 'dir' => [ - 'public' => 0755, -- 'private' => 0700, -+ 'private' => 0750, - ], - ], - ], diff --git a/services/tests/mastodon.nix b/services/tests/mastodon.nix index 244f0304..f5497520 100644 --- a/services/tests/mastodon.nix +++ b/services/tests/mastodon.nix @@ -42,7 +42,7 @@ let ''; in -pkgs.nixosTest { +{ name = "mastodon"; nodes = { diff --git a/services/tests/peertube.nix b/services/tests/peertube.nix index 27d79589..475c3f54 100644 --- a/services/tests/peertube.nix +++ b/services/tests/peertube.nix @@ -161,7 +161,7 @@ let ''; in -pkgs.nixosTest { +{ name = "peertube"; nodes = { diff --git a/services/tests/pixelfed-garage.nix b/services/tests/pixelfed-garage.nix index 13ad1ef7..66116774 100644 --- a/services/tests/pixelfed-garage.nix +++ b/services/tests/pixelfed-garage.nix @@ -114,7 +114,7 @@ let ${seleniumQuit}''; in -pkgs.nixosTest { +{ name = "test-pixelfed-garage"; nodes = {