Automated break down of Nix flake checks for CI (#512)

picking up from #492 to fix the CI issue.

note that i left the non-flake checks duplicating with flake ones (`pre-commit`, `panel`) in on both sides to get a sense of which way might be preferable.

Co-authored-by: Nicolas “Niols” Jeannerod <nicolas.jeannerod@moduscreate.com>
Reviewed-on: Fediversity/Fediversity#512

.forgejo/workflows/ci.yaml

@@ -22,94 +22,8 @@ jobs:
       - uses: actions/checkout@v4
       - run: nix-shell --run 'nix-unit ./deployment/data-model-test.nix'
 
-  check-mastodon:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.test-mastodon-service -L
-
-  check-peertube:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.test-peertube-service -L
-
   check-panel:
     runs-on: native
     steps:
       - uses: actions/checkout@v4
       - run: nix-build -A tests.panel
-
-  check-proxmox-basic:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.proxmox-basic -L
-
-  check-deployment-basic:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-basic -L
-
-  check-deployment-cli:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-cli -L
-
-  check-deployment-panel:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-panel -L
-
-  check-deployment-model:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-model-ssh -L
-
-  check-deployment-model-ssh:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-model-ssh -L
-
-  check-deployment-model-nixops4:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-model-nixops4 -L
-
-  check-deployment-model-tf:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix build .#checks.x86_64-linux.deployment-model-tf -L
-
-  ## NOTE: NixOps4 does not provide a good “dry run” mode, so we instead check
-  ## proxies for resources, namely whether their `.#vmOptions.<machine>` and
-  ## `.#nixosConfigurations.<machine>` outputs evaluate and build correctly, and
-  ## whether we can dry run `infra/proxmox-*.sh` on them. This will not catch
-  ## everything, and in particular not issues in how NixOps4 wires up the
-  ## resources, but that is still something.
-  check-resources:
-    runs-on: native
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          set -euC
-          echo ==================== [ VM Options ] ====================
-          machines=$(nix eval --impure --raw --expr 'with builtins; toString (attrNames (getFlake (toString ./.)).vmOptions)')
-          for machine in $machines; do
-            echo ~~~~~~~~~~~~~~~~~~~~~: $machine :~~~~~~~~~~~~~~~~~~~~~
-            nix build .#checks.x86_64-linux.vmOptions-$machine
-          done
-          echo
-          echo ==================== [ NixOS Configurations ] ====================
-          machines=$(nix eval --impure --raw --expr 'with builtins; toString (attrNames (getFlake (toString ./.)).nixosConfigurations)')
-          for machine in $machines; do
-            echo ~~~~~~~~~~~~~~~~~~~~~: $machine :~~~~~~~~~~~~~~~~~~~~~
-            nix build .#checks.x86_64-linux.nixosConfigurations-$machine
-          done

.forgejo/workflows/nix-flake-check.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+set -euC
+
+cd "$(dirname "$0")" || exit 3
+
+nix_eval () { nix eval --impure --raw --expr "with builtins; $1"; }
+system=$(nix_eval "currentSystem")
+checks=$(nix_eval "toJSON (attrNames (getFlake (toString ../..)).checks.$system)")
+
+output=$(mktemp)
+
+{
+    cat <<EOF
+name: Nix flake checks
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  push:
+    branches:
+      - main
+
+jobs:
+  _checks:
+    needs: $checks
+    runs-on: native
+    steps:
+      - run: true
+
+  _complete:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix-shell --run '.forgejo/workflows/nix-flake-check.sh check'
+EOF
+
+    for check in $(echo "$checks" | jq -r .[]); do
+        cat <<EOF
+
+  $check:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.$system.$check -vL
+EOF
+    done
+} >| "$output"
+
+target=$(basename "$0" .sh).yaml
+
+if [ $# -eq 1 ] && [ "$1" = "check" ]; then
+    if ! diff_output=$(diff --color=always "$target" "$output"); then
+        printf >&2 'Changes detected (\e[31m< current\e[0m | \e[32m> generated\e[0m):\n%s\n' "$diff_output"
+        exit 1
+    fi
+else
+    mv "$output" "$target"
+fi

.forgejo/workflows/nix-flake-check.yaml

@@ -0,0 +1,294 @@
+name: Nix flake checks
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  push:
+    branches:
+      - main
+
+jobs:
+  _checks:
+    needs: ["deployment-basic","deployment-cli","deployment-model-nixops4","deployment-model-ssh","deployment-model-tf","deployment-panel","nixops-deployment-providers-default","nixops-deployment-providers-fedi200","nixops-deployment-providers-fedi201","nixops-deployment-providers-forgejo-ci","nixops-deployment-providers-test","nixops-deployment-providers-vm02116","nixops-deployment-providers-vm02187","nixosConfigurations-fedi200","nixosConfigurations-fedi201","nixosConfigurations-forgejo-ci","nixosConfigurations-test01","nixosConfigurations-test02","nixosConfigurations-test03","nixosConfigurations-test04","nixosConfigurations-test05","nixosConfigurations-test06","nixosConfigurations-test11","nixosConfigurations-test12","nixosConfigurations-test13","nixosConfigurations-test14","nixosConfigurations-vm02116","nixosConfigurations-vm02187","panel","pre-commit","proxmox-basic","test-mastodon-service","test-peertube-service","vmOptions-fedi200","vmOptions-fedi201","vmOptions-test01","vmOptions-test02","vmOptions-test03","vmOptions-test04","vmOptions-test05","vmOptions-test06","vmOptions-test11","vmOptions-test12","vmOptions-test13","vmOptions-test14"]
+    runs-on: native
+    steps:
+      - run: true
+
+  _complete:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix-shell --run '.forgejo/workflows/nix-flake-check.sh check'
+
+  deployment-basic:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.deployment-basic -vL
+
+  deployment-cli:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.deployment-cli -vL
+
+  deployment-model-nixops4:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.deployment-model-nixops4 -vL
+
+  deployment-model-ssh:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.deployment-model-ssh -vL
+
+  deployment-model-tf:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.deployment-model-tf -vL
+
+  deployment-panel:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.deployment-panel -vL
+
+  nixops-deployment-providers-default:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-default -vL
+
+  nixops-deployment-providers-fedi200:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-fedi200 -vL
+
+  nixops-deployment-providers-fedi201:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-fedi201 -vL
+
+  nixops-deployment-providers-forgejo-ci:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-forgejo-ci -vL
+
+  nixops-deployment-providers-test:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-test -vL
+
+  nixops-deployment-providers-vm02116:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-vm02116 -vL
+
+  nixops-deployment-providers-vm02187:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixops-deployment-providers-vm02187 -vL
+
+  nixosConfigurations-fedi200:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-fedi200 -vL
+
+  nixosConfigurations-fedi201:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-fedi201 -vL
+
+  nixosConfigurations-forgejo-ci:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-forgejo-ci -vL
+
+  nixosConfigurations-test01:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test01 -vL
+
+  nixosConfigurations-test02:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test02 -vL
+
+  nixosConfigurations-test03:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test03 -vL
+
+  nixosConfigurations-test04:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test04 -vL
+
+  nixosConfigurations-test05:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test05 -vL
+
+  nixosConfigurations-test06:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test06 -vL
+
+  nixosConfigurations-test11:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test11 -vL
+
+  nixosConfigurations-test12:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test12 -vL
+
+  nixosConfigurations-test13:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test13 -vL
+
+  nixosConfigurations-test14:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-test14 -vL
+
+  nixosConfigurations-vm02116:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-vm02116 -vL
+
+  nixosConfigurations-vm02187:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.nixosConfigurations-vm02187 -vL
+
+  panel:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.panel -vL
+
+  pre-commit:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.pre-commit -vL
+
+  proxmox-basic:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.proxmox-basic -vL
+
+  test-mastodon-service:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.test-mastodon-service -vL
+
+  test-peertube-service:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.test-peertube-service -vL
+
+  vmOptions-fedi200:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-fedi200 -vL
+
+  vmOptions-fedi201:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-fedi201 -vL
+
+  vmOptions-test01:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test01 -vL
+
+  vmOptions-test02:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test02 -vL
+
+  vmOptions-test03:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test03 -vL
+
+  vmOptions-test04:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test04 -vL
+
+  vmOptions-test05:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test05 -vL
+
+  vmOptions-test06:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test06 -vL
+
+  vmOptions-test11:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test11 -vL
+
+  vmOptions-test12:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test12 -vL
+
+  vmOptions-test13:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test13 -vL
+
+  vmOptions-test14:
+    runs-on: native
+    steps:
+      - uses: actions/checkout@v4
+      - run: nix build .#checks.x86_64-linux.vmOptions-test14 -vL

default.nix

@@ -65,6 +65,7 @@ in
         pkgs.openssh
         pkgs.httpie
         pkgs.jq
+        pkgs.diffutils
         pkgs.nix-unit
         test-loop
         nixops4.packages.${system}.default

deployment/data-model-test.nix

@@ -98,8 +98,7 @@ in
                     {
                       options.enable = lib.mkEnableOption "Hello in the shell";
                     };
-                  implementation =
-                    cfg: {
+                  implementation = cfg: {
                     resources = lib.optionalAttrs cfg.enable {
                       hello.login-shell.packages.hello = pkgs.hello;
                     };

deployment/flake-part.nix

@@ -38,7 +38,7 @@
         };
 
         deployment-model-tf = import ./check/data-model-tf {
-         inherit (pkgs.testers) runNixOSTest;
+          inherit (pkgs.testers) runNixOSTest;
           inherit inputs sources;
         };
       };

services/tests/flake-part.nix

@@ -7,7 +7,6 @@
     {
       checks = {
         test-mastodon-service = pkgs.testers.runNixOSTest ./mastodon.nix;
-        test-pixelfed-garage-service = pkgs.testers.runNixOSTest ./pixelfed-garage.nix;
         test-peertube-service = pkgs.testers.runNixOSTest ./peertube.nix;
       };
     };