WIP: add attic cache, see #92

intending to document more about how our schemas work, i found our code currently buries this somewhere in `/nix/store`.
this change exposes that generated file in the project structure as well, facilitating inspection of our data schema in this more generic format.

Reviewed-on: Fediversity/Fediversity#351
Reviewed-by: Nicolas Jeannerod <nicolas.jeannerod@moduscreate.com>
Co-authored-by: Kiara Grouwstra <kiara@procolix.eu>
Co-committed-by: Kiara Grouwstra <kiara@procolix.eu>

.forgejo/workflows/update.yaml

@@ -0,0 +1,21 @@
+name: update-dependencies
+
+on:
+  workflow_dispatch: # allows manual triggering
+  schedule:
+    - cron: '0 0 1 * *' # monthly
+
+jobs:
+  lockfile:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+      - name: Install npins
+        run: nix profile install 'nixpkgs#npins'
+      - name: Update npins sources
+        uses: getchoo/update-npins@v0
+        with:
+          token: "${{ secrets.DEPLOY_KEY }}"

README.md

@@ -14,6 +14,12 @@ There already exist solutions for self-hosting, but they're not suitable for wha
 The ones we're aware of require substantial technical knowledge and time commitment by operators, especially for scaling to thousands of users.
 Not everyone has the expertise and time to run their own server.
 
+## Interactions
+
+To reach these goals, we aim to implement the following interactions between [actors](#actors) (depicted with rounded corners) and system components (see the [glossary](#glossary), depicted with rectangles).
+
+![](https://git.fediversity.eu/Fediversity/meta/raw/branch/main/architecture-docs/interactions.svg)
+
 ## Actors
 
 - Fediversity project team
@@ -57,11 +63,11 @@ Not everyone has the expertise and time to run their own server.
 
 - [Fediverse](https://en.wikipedia.org/wiki/Fediverse)
 
-  A collection of social networking services that can communicate with each other using a common protocol.
+  A collection of social networking applications that can communicate with each other using a common protocol.
 
-- Service
+- Application
 
-  A Fediverse application run by the hosting provider for an operator.
+  User-facing software (e.g. from Fediverse) run by the hosting provider for an operator.
 
 - Configuration
 
@@ -73,11 +79,11 @@ Not everyone has the expertise and time to run their own server.
 
   Make a resource, such as a virtual machine, available for use.
 
-  > Example: We use [Proxmox](https://www.proxmox.com) to provision VMs for services run by operators.
+  > Example: We use [Proxmox](https://www.proxmox.com) to provision VMs for applications run by operators.
 
 - Deploy
 
-  Put software, such as services, onto computers.
+  Put software, such as applications, onto computers.
   The software includes technical configuration that links software components.
   Most user-facing configuration remains untouched by the deployment process.
 
@@ -85,7 +91,7 @@ Not everyone has the expertise and time to run their own server.
 
 - Migrate
 
-  Move service configurations and user data to a different hosting provider.
+  Move service configurations and deployment state, including user data, from one hosting provider to another.
 
 - [NixOps4](https://github.com/nixops4/nixops4)
 
@@ -103,6 +109,18 @@ Not everyone has the expertise and time to run their own server.
 
   > Example: We need a resource provider for obtaining deployment secrets from a database.
 
+- Runtime backend
+
+  A type of digital environment one can run operating systems such as NixOS on, e.g. bare-metal, a hypervisor, or a container runtime.
+
+- Runtime environment
+
+  The thing a deployment runs on, an interface against which the deployment is working. See runtime backend.
+
+- Runtime config
+
+  Configuration logic specific to a runtime backend, e.g. how to deploy, how to access object storage.
+
 ## Development
 
 All the code made for this project is freely licenced under [EUPL](https://en.m.wikipedia.org/wiki/European_Union_Public_Licence).

infra/test-machines/test12/attic.nix

@@ -0,0 +1,203 @@
+{lib, tf, ...}:
+{
+  services.postgresql = {
+    enable = true;
+    authentication = lib.mkForce ''
+      local all all trust
+    '';
+    ensureDatabases = [
+      "atticd"
+    ];
+    ensureUsers = [
+      {
+        name = "atticd";
+        ensureDBOwnership = true;
+      }
+    ];
+  };
+
+  services.atticd = {
+    enable = true;
+    # one `monolithic` and any number of `api-server` nodes
+    mode = "monolithic";
+    environmentFile = "/var/lib/secrets/attic_env";
+    # https://github.com/zhaofengli/attic/blob/main/server/src/config-template.toml
+    settings = {
+      # Socket address to listen on
+      # listen = "[::]:8080";
+      # listen = "0.0.0.0:8080";
+      listen = "127.0.0.1:8080";
+
+      # Allowed `Host` headers
+      #
+      # This _must_ be configured for production use. If unconfigured or the
+      # list is empty, all `Host` headers are allowed.
+      allowed-hosts = [];
+
+      # The canonical API endpoint of this server
+      #
+      # This is the endpoint exposed to clients in `cache-config` responses.
+      #
+      # This _must_ be configured for production use. If not configured, the
+      # API endpoint is synthesized from the client's `Host` header which may
+      # be insecure.
+      #
+      # The API endpoint _must_ end with a slash (e.g., `https://domain.tld/attic/`
+      # not `https://domain.tld/attic`).
+      api-endpoint = "https://${tf.resource.hetznerdns_zone.main.name}/";
+
+      # Whether to soft-delete caches
+      #
+      # If this is enabled, caches are soft-deleted instead of actually
+      # removed from the database. Note that soft-deleted caches cannot
+      # have their names reused as long as the original database records
+      # are there.
+      #soft-delete-caches = false;
+
+      # Whether to require fully uploading a NAR if it exists in the global cache.
+      #
+      # If set to false, simply knowing the NAR hash is enough for
+      # an uploader to gain access to an existing NAR in the global
+      # cache.
+      #require-proof-of-possession = true;
+
+      # Database connection
+      database = {
+        # Connection URL
+        #
+        # For production use it's recommended to use PostgreSQL.
+        # url = "postgresql:///atticd:password@127.0.0.1:5432/atticd";
+        url = "postgresql:///atticd?host=/run/postgresql";
+
+        # Whether to enable sending on periodic heartbeat queries
+        #
+        # If enabled, a heartbeat query will be sent every minute
+        #heartbeat = false;
+      };
+
+      # File storage configuration
+      storage = {
+        # Storage type
+        #
+        # Can be "local" or "s3".
+        type = "s3";
+
+        # ## Local storage
+
+        # The directory to store all files under
+        # path = "%storage_path%";
+
+        # ## S3 Storage (set type to "s3" and uncomment below)
+
+        # The AWS region
+        region = tf.resource.cloudflare_r2_bucket.atticd.location;  # is this even used for R2?
+
+        # The name of the bucket
+        bucket = tf.resource.cloudflare_r2_bucket.atticd.name;
+
+        # Custom S3 endpoint
+        #
+        # Set this if you are using an S3-compatible object storage (e.g., Minio).
+        endpoint = "https://2b56368370c7a8e7f41328f0b8d4040a.r2.cloudflarestorage.com";
+
+        # Credentials
+        #
+        # If unset, the credentials are read from the `AWS_ACCESS_KEY_ID` and
+        # `AWS_SECRET_ACCESS_KEY` environment variables.
+        # storage.credentials = {
+        #   access_key_id = "";
+        #   secret_access_key = "";
+        # };
+      };
+
+      # Data chunking
+      #
+      # Warning: If you change any of the values here, it will be
+      # difficult to reuse existing chunks for newly-uploaded NARs
+      # since the cutpoints will be different. As a result, the
+      # deduplication ratio will suffer for a while after the change.
+      chunking = {
+        # The minimum NAR size to trigger chunking
+        #
+        # If 0, chunking is disabled entirely for newly-uploaded NARs.
+        # If 1, all NARs are chunked.
+        nar-size-threshold = 65536; # chunk files that are 64 KiB or larger
+
+        # The preferred minimum size of a chunk, in bytes
+        min-size = 16384;            # 16 KiB
+
+        # The preferred average size of a chunk, in bytes
+        avg-size = 65536;            # 64 KiB
+
+        # The preferred maximum size of a chunk, in bytes
+        max-size = 262144;           # 256 KiB
+      };
+
+      # Compression
+      compression = {
+        # Compression type
+        #
+        # Can be "none", "brotli", "zstd", or "xz"
+        type = "zstd";
+
+        # Compression level
+        #level = 8;
+      };
+
+      # Garbage collection
+      garbage-collection = {
+        # The frequency to run garbage collection at
+        #
+        # By default it's 12 hours. You can use natural language
+        # to specify the interval, like "1 day".
+        #
+        # If zero, automatic garbage collection is disabled, but
+        # it can still be run manually with `atticd --mode garbage-collector-once`.
+        interval = "12 hours";
+
+        # Default retention period
+        #
+        # Zero (default) means time-based garbage-collection is
+        # disabled by default. You can enable it on a per-cache basis.
+        #default-retention-period = "6 months";
+      };
+
+      jwt = {
+        # WARNING: Changing _anything_ in this section will break any existing
+        # tokens. If you need to regenerate them, ensure that you use the the
+        # correct secret and include the `iss` and `aud` claims.
+
+        # JWT `iss` claim
+        #
+        # Set this to the JWT issuer that you want to validate.
+        # If this is set, all received JWTs will validate that the `iss` claim
+        # matches this value.
+        #token-bound-issuer = "some-issuer";
+
+        # JWT `aud` claim
+        #
+        # Set this to the JWT audience(s) that you want to validate.
+        # If this is set, all received JWTs will validate that the `aud` claim
+        # contains at least one of these values.
+        #token-bound-audiences = ["some-audience1", "some-audience2"];
+      };
+
+      # jwt.signing = {
+      #   # JWT RS256 secret key
+      #   #
+      #   # Set this to the base64-encoded private half of an RSA PEM PKCS1 key.
+      #   # TODO
+      #   # You can also set it via the `ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64`
+      #   # environment variable.
+      #   token-rs256-secret-base64 = "%token_rs256_secret_base64%";
+
+      #   # JWT HS256 secret key
+      #   #
+      #   # Set this to the base64-encoded HMAC secret key.
+      #   # You can also set it via the `ATTIC_SERVER_TOKEN_HS256_SECRET_BASE64`
+      #   # environment variable.
+      #   #token-hs256-secret-base64 = "";
+      # };
+    };
+  };
+}

infra/test-machines/test12/default.nix

@@ -16,4 +16,10 @@
       gateway = "2a00:51c0:13:1305::1";
     };
   };
+
+  nixos.module = {
+    imports = [
+      ./attic.nix
+    ];
+  };
 }

panel/.gitignore

@@ -11,5 +11,5 @@ db.sqlite3
 src/db.sqlite3
 src/static
 src/panel/static/htmx*
-src/panel/configuration/schema.py
+src/panel/configuration/schema.*
 .credentials

panel/nix/package.nix

@@ -14,7 +14,7 @@ let
       root = ../src;
       fileset = intersection (gitTracked ../../.) ../src;
     };
-  pyproject = with lib; fromTOML pyproject-toml;
+  pyproject = fromTOML pyproject-toml;
   # TODO: define this globally
   name = "panel";
   # TODO: we may want this in a file so it's easier to read statically
@@ -29,28 +29,35 @@ let
     include-package-data = true
   '';
 
-  generated = [
+  generated =
-    {
+    let
-      from = "${sources.htmx}/dist/htmx.min.js";
+      jsonschema = callPackage "${sources.clan-core}/lib/jsonschema" { } { };
-      to = "./panel/static/htmx.min.js";
+      frontend-options = jsonschema.parseModule ../../deployment/options.nix;
-    }
+      schema = with builtins; toFile "schema.json" (toJSON frontend-options);
-    {
+    in
-      from =
+    [
-        let
+      {
-          jsonschema = callPackage "${sources.clan-core}/lib/jsonschema" { } { };
+        from = "${sources.htmx}/dist/htmx.min.js";
-          frontend-options = jsonschema.parseModule ../../deployment/options.nix;
+        to = "./panel/static/htmx.min.js";
-          schema = with builtins; toFile "schema.json" (toJSON frontend-options);
+      }
-          codegen = "${python3Packages.datamodel-code-generator}/bin/datamodel-codegen";
+      {
-          pydantic = runCommand "schema.py" { } ''
+        from = schema;
-            # replace plain `pydantic` with `drf_pydantic` so we can create forms automatically
+        to = "./panel/configuration/schema.json";
-            ${codegen} --input ${schema} | sed '/from pydantic/a\
+      }
-            from drf_pydantic import BaseModel' > $out
+      {
-          '';
+        from =
-        in
+          let
-        "${pydantic}";
+            codegen = "${python3Packages.datamodel-code-generator}/bin/datamodel-codegen";
-      to = "./panel/configuration/schema.py";
+            pydantic = runCommand "schema.py" { } ''
-    }
+              # replace plain `pydantic` with `drf_pydantic` so we can create forms automatically
-  ];
+              ${codegen} --input ${schema} | sed '/from pydantic/a\
+              from drf_pydantic import BaseModel' > $out
+            '';
+          in
+          "${pydantic}";
+        to = "./panel/configuration/schema.py";
+      }
+    ];
 in
 python3.pkgs.buildPythonPackage {
   pname = name;