bash-based deployment

deduplicate flake inputs

make re-exports explicit again

Revert "deduplicate flake inputs"

This reverts commit 95769084ce.

switch launch shell to root flake's nixpkgs, see #279

use flake-sourced nixos-anywhere in tf, to reproduce modules for nix

properly pass repo dir for prod, be it with hard-coded TF init

move tf init out of python over read-only nix env

skip tf lock in views.py over read-only nix env

specify XDG_CACHE_HOME, workaround to error writing to /var/empty/.cache

update

document updating TF module

get TF in prod to the same 'installable ... does not correspond to a Nix language value' for non-flakes

seemingly gets further when a similar command is tried from terminal.
as per https://github.com/NixOS/nix/issues/8752#issuecomment-1694714693,
this may have to do with aligning the current working directory.

rm launch flake, as i seem to have reached similar progress without it

update nixos-anywhere to fix error 'installable ... does not correspond to a Nix language value'

rm comment

untrack TF generated provider/module stuff - local dev now requires following launch/README.md

for now gitignore .auto.tfvars.json used to track TF module of nixos-anywhere

in case we want that file for something else, we can move this (and its
ignore) to something separate.

use a mutable HOME in TF for nixos-anywhere to make a `.ssh` dir in - will this not backfire?

change ssh user to root

allow accessing test vms from fedi201's machine ssh key, closes #286

allow accessing test vms from fedi201's machine ssh key, closes #286

update nixpkgs to unstable - resolves manual deploy error on bootloader already on newer version

switch to bash deployment

tmp

infra/machines/fedi201/fedipanel.nix

@@ -19,6 +19,24 @@ in
     enable = true;
     production = true;
     domain = "demo.fediversity.eu";
+    # FIXME: make it work without this duplication
+    settings =
+      let
+        cfg = config.services.${name};
+      in
+      {
+        STATIC_ROOT = "/var/lib/${name}/static";
+        DEBUG = false;
+        ALLOWED_HOSTS = [
+          cfg.domain
+          cfg.host
+          "localhost"
+          "[::1]"
+        ];
+        CSRF_TRUSTED_ORIGINS = [ "https://${cfg.domain}" ];
+        COMPRESS_OFFLINE = true;
+        LIBSASS_OUTPUT_STYLE = "compressed";
+      };
     secrets = {
       SECRET_KEY = config.age.secrets.panel-secret-key.path;
     };

launch/.envrc

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
+# shellcheck shell=bash
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv --flake .)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use flake
+fi

launch/.terraform.lock.hcl

@@ -0,0 +1,16 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/hashicorp/external" {
+  version = "2.3.4"
+  hashes = [
+    "h1:HfVaWMC7Tz+tRfoWZtGCX2MATcgX3HsexoirWdi/voo=",
+  ]
+}
+
+provider "registry.opentofu.org/hashicorp/null" {
+  version = "3.2.3"
+  hashes = [
+    "h1:qTlGDGC3RmXIPLgwsIh4LHG/DrAR6T6L+Wn6egnQnwE=",
+  ]
+}

launch/deploy.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -uex -o pipefail
+declare domain hostname config initialUser
+eval "$(jq -r '@sh "domain=\(.domain) hostname=\(.hostname) config=\(.config) initialUser=\(.initialUser)"')"
+TARGET_HOST="${hostname}.abundos.eu"
+TARGET="root@${TARGET_HOST}"
+wrapper="$(mktemp -d)/wrapper.nix"
+echo "(import $(readlink -f "./${config}.nix")).extendModules { specialArgs.terraform = { domain = \"${domain}\"; hostname = \"${hostname}\"; initialUser = builtins.fromJSON ''${initialUser}''; }; }" > "$wrapper"
+NIXOS_SYSTEM=$(nix build --no-link --json --option show-trace true --file "$wrapper" "config.system.build.toplevel" | jq -r '.[].outputs.out')
+sshOpts=(-p 22 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no)
+NIX_SSHOPTS="${sshOpts[*]}" nix copy -s --experimental-features nix-command --to "ssh://$TARGET" "$NIXOS_SYSTEM"
+switchCommand="nix-env -p /nix/var/nix/profiles/system --set $(printf "%q" "$NIXOS_SYSTEM"); /nix/var/nix/profiles/system/bin/switch-to-configuration switch"
+deploy_status=0
+# shellcheck disable=SC2029
+ssh "${sshOpts[@]}" "$TARGET" "$switchCommand" || deploy_status="$?"
+exit "$deploy_status"

launch/env.nix

@@ -0,0 +1,11 @@
+{
+  pkgs,
+  ...
+}:
+pkgs.stdenv.mkDerivation {
+  name = "fediversity-repo";
+  src = ../.;
+  installPhase = ''
+    cp -r . $out
+  '';
+}

launch/garage.nix

@@ -0,0 +1,37 @@
+let
+  ## NOTE: All of these secrets are publicly available in this source file
+  ## and will end up in the Nix store. We don't care as they are only ever
+  ## used for testing anyway.
+  ##
+  ## FIXME: Generate and store in NixOps4's state.
+  mastodonS3KeyConfig =
+    { pkgs, ... }:
+    {
+      s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558";
+      s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34";
+    };
+  peertubeS3KeyConfig =
+    { pkgs, ... }:
+    {
+      s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b";
+      s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395";
+    };
+  pixelfedS3KeyConfig =
+    { pkgs, ... }:
+    {
+      s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b";
+      s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987";
+    };
+in
+import ./shared.nix {
+  module =
+    { pkgs, ... }:
+    {
+      fediversity = {
+        garage.enable = true;
+        pixelfed = pixelfedS3KeyConfig { inherit pkgs; };
+        mastodon = mastodonS3KeyConfig { inherit pkgs; };
+        peertube = peertubeS3KeyConfig { inherit pkgs; };
+      };
+    };
+}

launch/mastodon.nix

@@ -0,0 +1,20 @@
+let
+  mastodonS3KeyConfig =
+    { pkgs, ... }:
+    {
+      s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK3515373e4c851ebaad366558";
+      s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34";
+    };
+in
+import ./shared.nix {
+  module =
+    { pkgs, ... }:
+    {
+      fediversity = {
+        mastodon = mastodonS3KeyConfig { inherit pkgs; } // {
+          enable = true;
+        };
+        temp.cores = 1; # FIXME: should come from NixOps4 eventually
+      };
+    };
+}

launch/peertube.nix

@@ -0,0 +1,23 @@
+let
+  peertubeS3KeyConfig =
+    { pkgs, ... }:
+    {
+      s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GK1f9feea9960f6f95ff404c9b";
+      s3SecretKeyFile = pkgs.writeText "s3SecretKey" "7295c4201966a02c2c3d25b5cea4a5ff782966a2415e3a196f91924631191395";
+    };
+in
+import ./shared.nix {
+  module =
+    { pkgs, ... }:
+    {
+      fediversity = {
+        peertube = peertubeS3KeyConfig { inherit pkgs; } // {
+          enable = true;
+          ## NOTE: Only ever used for testing anyway.
+          ##
+          ## FIXME: Generate and store in NixOps4's state.
+          secretsFile = pkgs.writeText "secret" "574e093907d1157ac0f8e760a6deb1035402003af5763135bae9cbd6abe32b24";
+        };
+      };
+    };
+}

launch/pixelfed.nix

@@ -0,0 +1,19 @@
+let
+  pixelfedS3KeyConfig =
+    { pkgs, ... }:
+    {
+      s3AccessKeyFile = pkgs.writeText "s3AccessKey" "GKb5615457d44214411e673b7b";
+      s3SecretKeyFile = pkgs.writeText "s3SecretKey" "5be6799a88ca9b9d813d1a806b64f15efa49482dbe15339ddfaf7f19cf434987";
+    };
+in
+import ./shared.nix {
+  module =
+    { pkgs, ... }:
+    {
+      fediversity = {
+        pixelfed = pixelfedS3KeyConfig { inherit pkgs; } // {
+          enable = true;
+        };
+      };
+    };
+}

launch/resource.nix

@@ -0,0 +1,41 @@
+{
+  lib,
+  config,
+  ...
+}:
+
+let
+  inherit (lib) attrValues elem mkDefault;
+  inherit (lib.attrsets) concatMapAttrs optionalAttrs;
+  inherit (lib.strings) removeSuffix;
+
+  secretsPrefix = ../secrets;
+  secrets = import (secretsPrefix + "/secrets.nix");
+  keys = import ../keys;
+
+in
+{
+  fediversityVm.hostPublicKey = mkDefault keys.systems.${config.fediversityVm.name};
+
+  ## The configuration of the machine. We strive to keep in this file only the
+  ## options that really need to be injected from the resource. Everything else
+  ## should go into the `./nixos` subdirectory.
+  imports = [
+    ../infra/common/options.nix
+    ../infra/common/nixos
+  ];
+
+  ## Read all the secrets, filter the ones that are supposed to be readable
+  ## with this host's public key, and add them correctly to the configuration
+  ## as `age.secrets.<name>.file`.
+  age.secrets = concatMapAttrs (
+    name: secret:
+    optionalAttrs (elem config.fediversityVm.hostPublicKey secret.publicKeys) {
+      ${removeSuffix ".age" name}.file = secretsPrefix + "/${name}";
+    }
+  ) secrets;
+
+  ## FIXME: Remove direct root authentication once the NixOps4 NixOS provider
+  ## supports users with password-less sudo.
+  users.users.root.openssh.authorizedKeys.keys = attrValues keys.contributors;
+}

launch/shared.nix

@@ -0,0 +1,45 @@
+{
+  system ? "x86_64-linux",
+  sources ? import ../npins,
+  pkgs ? import sources.nixpkgs {
+    inherit system;
+    config = { };
+    overlays = [ (import ../panel/nix/overlay.nix) ];
+  },
+  module,
+  ...
+}:
+import "${sources.nixpkgs}/nixos/lib/eval-config.nix" {
+  modules = [
+    "${sources.disko}/module.nix"
+    "${sources.agenix}/modules/age.nix"
+    ../services/fediversity
+    ./resource.nix
+    # FIXME: get VM details from TF
+    module
+    (
+      {
+        terraform,
+        ...
+      }:
+      let
+        inherit (terraform) hostname;
+      in
+      {
+        imports = [
+          ../infra/test-machines/${hostname}
+        ];
+        fediversityVm.name = hostname;
+        fediversity = {
+          inherit (terraform) domain;
+          temp.initialUser = {
+            inherit (terraform.initialUser) username email displayName;
+            # FIXME: disgusting, but nvm, this is going to be replaced by
+            # proper central authentication at some point
+            passwordFile = pkgs.writeText "password" terraform.initialUser.password;
+          };
+        };
+      }
+    )
+  ];
+}

npins/sources.json

@@ -1,5 +1,44 @@
 {
   "pins": {
+    "agenix": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "ryantm",
+        "repo": "agenix"
+      },
+      "branch": "main",
+      "revision": "e600439ec4c273cf11e06fe4d9d906fb98fa097c",
+      "url": "https://github.com/ryantm/agenix/archive/e600439ec4c273cf11e06fe4d9d906fb98fa097c.tar.gz",
+      "hash": "006ngydiykjgqs85cl19h9klq8kaqm5zs0ng51dnwy7nzgqxzsdr"
+    },
+    "disko": {
+      "type": "GitRelease",
+      "repository": {
+        "type": "GitHub",
+        "owner": "nix-community",
+        "repo": "disko"
+      },
+      "pre_releases": false,
+      "version_upper_bound": null,
+      "release_prefix": null,
+      "version": "v1.11.0",
+      "revision": "cdf8deded8813edfa6e65544f69fdd3a59fa2bb4",
+      "url": "https://api.github.com/repos/nix-community/disko/tarball/v1.11.0",
+      "hash": "13brimg7z7k9y36n4jc1pssqyw94nd8qvgfjv53z66lv4xkhin92"
+    },
+    "flake-inputs": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "fricklerhandwerk",
+        "repo": "flake-inputs"
+      },
+      "branch": "main",
+      "revision": "559574c9cbb8af262f3944b67d60fbf0f6ad03c3",
+      "url": "https://github.com/fricklerhandwerk/flake-inputs/archive/559574c9cbb8af262f3944b67d60fbf0f6ad03c3.tar.gz",
+      "hash": "0gbhmp6x2vdzvfnsvqzal3g8f8hx2ia6r73aibc78kazf78m67x6"
+    },
     "htmx": {
       "type": "GitRelease",
       "repository": {
@@ -27,11 +66,23 @@
       "url": "https://github.com/nix-community/nix-unit/archive/2071bbb765681ac3d8194ec560c8b27ff2a3b541.tar.gz",
       "hash": "0blz1kcmn9vnr9q3iqp2mv13hv3pdccljmmc54f8j7ybf5v0wgmp"
     },
+    "nixos-anywhere": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "KiaraGrouwstra",
+        "repo": "nixos-anywhere"
+      },
+      "branch": "special-args-nested-flake-fixed",
+      "revision": "5aa35145f045eb23fa8773821d5626bcf54dbe0e",
+      "url": "https://github.com/KiaraGrouwstra/nixos-anywhere/archive/5aa35145f045eb23fa8773821d5626bcf54dbe0e.tar.gz",
+      "hash": "0m67iyd04wl183il1cfi623xpxcvbbpc5x1gh74478qc3fgr0g54"
+    },
     "nixpkgs": {
       "type": "Channel",
       "name": "nixpkgs-unstable",
-      "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre711046.8edf06bea5bc/nixexprs.tar.xz",
-      "hash": "1mwsn0rvfm603svrq3pca4c51zlix5gkyr4gl6pxhhq3q6xs5s8y"
+      "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.05pre777917.b7ba7f9f45c5/nixexprs.tar.xz",
+      "hash": "0jb6b7sv66bn06pchj2l88z0i5dlz0c2vb3z6pjjlq2p8q11zigg"
     }
   },
   "version": 3

panel/default.nix

@@ -20,11 +20,18 @@ in
       pkgs.npins
       manage
     ];
-    env = import ./env.nix { inherit lib pkgs; } // {
-      NPINS_DIRECTORY = toString ../npins;
-      CREDENTIALS_DIRECTORY = toString ./.credentials;
-      DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3";
-    };
+    env =
+      let
+        inherit (builtins) toString;
+      in
+      import ./env.nix { inherit lib pkgs; }
+      // {
+        NPINS_DIRECTORY = toString ../npins;
+        CREDENTIALS_DIRECTORY = toString ./.credentials;
+        DATABASE_URL = "sqlite:///${toString ./src}/db.sqlite3";
+        # locally: use a fixed relative reference, so we can use our newest files without copying to the store
+        REPO_DIR = toString ../.;
+      };
     shellHook = ''
       ln -sf ${sources.htmx}/dist/htmx.js src/panel/static/htmx.min.js
       # in production, secrets are passed via CREDENTIALS_DIRECTORY by systemd.

panel/env.nix

@@ -3,16 +3,13 @@
   pkgs,
   ...
 }:
-let
-  inherit (builtins) toString;
-in
 {
-  REPO_DIR = toString ../.;
-  # explicitly use nix, as e.g. lix does not have configurable-impure-env
   BIN_PATH = lib.makeBinPath [
-    # explicitly use nix, as e.g. lix does not have configurable-impure-env
-    pkgs.nix
-    # nixops error maybe due to our flake git hook: executing 'git': No such file or directory
+    pkgs.lix
+    pkgs.bash
+    pkgs.coreutils
+    pkgs.openssh
     pkgs.git
+    pkgs.jq # implicit dep of nixos-anywhere TF: https://github.com/nix-community/nixos-anywhere/issues/416
   ];
 }

panel/nix/configuration.nix

@@ -29,6 +29,7 @@ let
       ((pkgs.formats.pythonVars { }).generate "settings.py" cfg.settings)
       (builtins.toFile "extra-settings.py" cfg.extra-settings)
     ];
+    REPO_DIR = import ../../launch/env.nix { inherit lib pkgs; };
   };
 
   python-environment = pkgs.python3.withPackages (

panel/nix/package.nix

@@ -1,5 +1,6 @@
 {
   lib,
+  pkgs,
   sqlite,
   python3,
   sources ? import ../../npins,
@@ -11,7 +12,7 @@ let
       root = ../src;
       fileset = intersection (gitTracked ../../.) ../src;
     };
-  pyproject = with lib; fromTOML pyproject-toml;
+  pyproject = fromTOML pyproject-toml;
   # TODO: define this globally
   name = "panel";
   # TODO: we may want this in a file so it's easier to read statically
@@ -58,7 +59,9 @@ python3.pkgs.buildPythonPackage {
     mkdir -p $out/bin
     cp -v ${src}/manage.py $out/bin/manage.py
     chmod +x $out/bin/manage.py
-    wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
+    wrapProgram $out/bin/manage.py \
+      --set REPO_DIR "${import ../../launch/env.nix { inherit lib pkgs; }}" \
+      --prefix PYTHONPATH : "$PYTHONPATH"
     cp ${sources.htmx}/dist/htmx.min.js* $out/${python3.sitePackages}/panel/static/
   '';
 }

panel/src/panel/settings.py

@@ -10,7 +10,9 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 
+import re
 import sys
+import subprocess
 import os
 import importlib.util
 import dj_database_url
@@ -18,6 +20,8 @@ import dj_database_url
 from os import environ as env
 from pathlib import Path
 
+STORE_PATTERN = re.compile("^/nix/store/[^/]+$")
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -171,6 +175,54 @@ COMPRESS_PRECOMPILERS = [
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "filters": {
+        "require_debug_false": {
+            "()": "django.utils.log.RequireDebugFalse",
+        },
+        "require_debug_true": {
+            "()": "django.utils.log.RequireDebugTrue",
+        },
+    },
+    "formatters": {
+        "django.server": {
+            "()": "django.utils.log.ServerFormatter",
+            "format": "[{server_time}] {message}",
+            "style": "{",
+        }
+    },
+    "handlers": {
+        "console": {
+            "level": "INFO",
+            # "filters": ["require_debug_true"],
+            "class": "logging.StreamHandler",
+        },
+        "django.server": {
+            "level": "INFO",
+            "class": "logging.StreamHandler",
+            "formatter": "django.server",
+        },
+        "mail_admins": {
+            "level": "ERROR",
+            "filters": ["require_debug_false"],
+            "class": "django.utils.log.AdminEmailHandler",
+        },
+    },
+    "loggers": {
+        "django": {
+            "handlers": ["console", "mail_admins"],
+            "level": "INFO",
+        },
+        "django.server": {
+            "handlers": ["django.server"],
+            "level": "INFO",
+            "propagate": False,
+        },
+    },
+}
+
 # Customization via user settings
 # This must be at the end, as it must be able to override the above
 # TODO(@fricklerhandwerk):

panel/src/panel/views.py

@@ -1,5 +1,6 @@
 from enum import Enum
 import json
+import logging
 import subprocess
 import os
 
@@ -13,6 +14,8 @@ from django.shortcuts import render
 from panel import models, settings
 from panel.configuration import forms
 
+logger = logging.getLogger(__name__)
+
 
 class Index(TemplateView):
     template_name = 'index.html'
@@ -102,7 +105,7 @@ class DeploymentStatus(ConfigurationForm):
         # Check for deploy button
         if "deploy" in self.request.POST.keys():
             deployment_result, deployment_params = self.deployment(obj)
-            deployment_succeeded = deployment_result.returncode == 0
+            deployment_succeeded = deployment_result == 0
 
         return render(self.request, "partials/deployment_result.html", {
             "deployment_succeeded": deployment_succeeded,
@@ -123,36 +126,62 @@ class DeploymentStatus(ConfigurationForm):
             })
 
     def deployment(self, obj):
-        submission = obj.parsed_value.model_dump_json()
+        testvms = {
+            "garage": "test01",
+            "mastodon": "test06",
+            "pixelfed": "test04",
+            "peertube": "test03",
+        }
+        submission = json.loads(obj.parsed_value.model_dump_json())
+        logger.debug(f"submission: {submission}")
         # FIXME: let the user specify these from the form (#190)
         dummy_user = {
-            "initialUser": {
+            "initialUser": json.dumps({
                 "displayName": "Testy McTestface",
                 "username": "test",
                 "password": "testtest",
                 "email": "test@test.com",
-            },
+            }),
         }
-        # serialize back and forth now we still need to manually inject the dummy user
-        deployment_params = json.dumps(dummy_user | json.loads(submission))
-        env = {
-            "PATH": settings.bin_path,
-            # pass in form info to our deployment
-            "DEPLOYMENT": deployment_params,
-        }
-        cmd = [
-            "nix",
-            "develop",
-            "--extra-experimental-features",
-            "configurable-impure-env",
-            "--command",
-            "nixops4",
-            "apply",
-            "test",
-        ]
-        deployment_result = subprocess.run(
-            cmd,
-            cwd=settings.repo_dir,
-            env=env,
-        )
+        deployment_result = 0
+        if submission["enable"]:
+            services = [
+                "mastodon",
+                "pixelfed",
+                "peertube",
+            ]
+            for service in [
+                "garage",
+            ] + services:
+                if (service == "garage" and any(submission[srvc]["enable"] for srvc in services)) or submission[service]["enable"]:
+                    hostname = testvms[service]
+                    # serialize back and forth now we still need to manually inject the dummy user
+                    deployment_params = json.dumps({
+                        "config": service,
+                        "hostname": hostname,
+                        "initialUser": json.dumps({
+                            "displayName": "Testy McTestface",
+                            "username": "test",
+                            "password": "testtest",
+                            "email": "test@test.com",
+                        }),
+                    } | submission)
+                    logger.debug(f"deployment_params: {deployment_params}")
+                    deployment_result = deployment_result or subprocess.run(
+                        ["./deploy.sh"],
+                        cwd=f"{settings.repo_dir}/launch",
+                        env={
+                            "PATH": settings.bin_path,
+                            # warning: ignoring error initializing Lix fetcher cache: error: creating directory '/var/empty/.cache': Operation not permitted
+                            "HOME": "/tmp",
+                        },
+                        # pass in form info to our deployment
+                        input=deployment_params,
+                        text=True,
+                    ).returncode
+            else:
+                logger.debug(f"service {service} disabled")
+        else:
+            # FIXME: implement disable
+            logger.debug("deployment disabled")
         return deployment_result, json.loads(deployment_params)