From 84ba26d18784961b90d599f40b8f55fadd917772 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 11:30:32 +0100
Subject: [PATCH 1/7] Move Proxmox-related things under `deployment/proxmox`

---
 .../proxmox/README.org                           | 16 ++++++++--------
 .../{provision-vm.sh => proxmox/provision.sh}    | 14 ++++----------
 2 files changed, 12 insertions(+), 18 deletions(-)
 rename proxmox/proxmox-provisioning.org => deployment/proxmox/README.org (90%)
 rename deployment/{provision-vm.sh => proxmox/provision.sh} (94%)

diff --git a/proxmox/proxmox-provisioning.org b/deployment/proxmox/README.org
similarity index 90%
rename from proxmox/proxmox-provisioning.org
rename to deployment/proxmox/README.org
index 94f6cc0c..7776aa44 100644
--- a/proxmox/proxmox-provisioning.org
+++ b/deployment/proxmox/README.org
@@ -1,12 +1,12 @@
-#+title: Provisioning a Proxmox VM
-#+author: Kevin Muller, Hans van Zijst & Nicolas Jeannerod
-#+date: <2024-10-25 Fri>
+#+title: Provisioning VMs via Proxmox
 
-* Fediversity Proxmox
-- http://192.168.51.81:8006/.
-- It is only accessible via Procolix's VPN; see with Kevin.
-- You will need identifiers. Also see with Kevin. Select “Promox VE authentication server”.
-- Ignore “You do not have a valid subscription” message.
+* Quick links
+- Proxmox API doc :: https://pve.proxmox.com/pve-docs/api-viewer
+- Fediversity Proxmox ::
+  - http://192.168.51.81:8006/.
+  - It is only accessible via Procolix's VPN; see with Kevin.
+  - You will need identifiers. Also see with Kevin. Select “Promox VE authentication server”.
+  - Ignore “You do not have a valid subscription” message.
 * Basic terminology
 - Node :: physical host
 * Preparing the machine configuration
diff --git a/deployment/provision-vm.sh b/deployment/proxmox/provision.sh
similarity index 94%
rename from deployment/provision-vm.sh
rename to deployment/proxmox/provision.sh
index c624fd79..39d74ac1 100755
--- a/deployment/provision-vm.sh
+++ b/deployment/proxmox/provision.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env sh
 set -euC
 
-## Proxmox API doc: https://pve.proxmox.com/pve-docs/api-viewer
-
 ################################################################################
 ## Parse arguments
 
@@ -24,7 +22,7 @@ Required:
   --vmid INT        Identifier of the VM
 
   If not provided via the command line, username and password will be looked for
-  in a `.proxmox` file in the current working directory, the username on the
+  in a '.proxmox' file in the current working directory, the username on the
   first line, and the password on the second.
 
 Optional:
@@ -68,11 +66,7 @@ fi
 
 [ -z "$vmid" ] && die 'Required: `--vmid`.\n'
 
-printf 'Configuration:\n'
-
-printf '  username: %s\n' $username
-printf '  password: %s\n' $password
-printf '  vmid: %s\n' $vmid
+printf 'Provisioning VM %d with:\n' $vmid
 
 readonly iso
 readonly sockets
@@ -205,7 +199,7 @@ while :; do
   esac
 done
 
-printf 'done.\n'
+printf ' done.\n'
 
 ################################################################################
 ## Start VM
@@ -220,4 +214,4 @@ wait_ $(from_response .data)
 http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
 wait_ $(from_response .data)
 
-printf 'done.\n'
+printf ' done.\n'

From 1c614ff3b8c0af739ad06474d51429c24c44f07a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 11:43:49 +0100
Subject: [PATCH 2/7] Add VM removal script

---
 deployment/proxmox/remove.sh | 137 +++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100755 deployment/proxmox/remove.sh

diff --git a/deployment/proxmox/remove.sh b/deployment/proxmox/remove.sh
new file mode 100755
index 00000000..259eeb12
--- /dev/null
+++ b/deployment/proxmox/remove.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env sh
+set -euC
+
+################################################################################
+## Constants
+
+readonly apiurl=https://192.168.51.81:8006/api2/json
+
+## FIXME: There seems to be a problem with file upload where the task is
+## registered to `node051` no matter what node we are actually uploading to? For
+## now, let us just use `node051` everywhere.
+readonly node=node051
+
+################################################################################
+## Parse arguments
+
+username=
+password=
+vmids=
+
+help () {
+  cat <<EOF
+Usage: $0 [OPTION...] [ID...]
+
+Authentication options:
+  --username STR    Username, with provider (eg. niols@pve)
+  --password STR    Password
+
+  If not provided via the command line, username and password will be looked for
+  in a '.proxmox' file in the current working directory, the username on the
+  first line, and the password on the second.
+
+Others:
+  -h|-?|--help      Show this help and exit
+EOF
+}
+
+die () { printf "$@"; printf '\n'; help; exit 2; }
+
+while [ $# -gt 0 ]; do
+  argument=$1
+  shift
+  case $argument in
+    --username) readonly username=$1; shift ;;
+    --password) readonly password=$1; shift ;;
+
+    -h|-\?|--help) help; exit 0 ;;
+
+    -*) die 'Unknown argument: `%s`.' "$argument" ;;
+
+    *) vmids="$vmids $argument" ;;
+  esac
+done
+
+if [ -z "$username" ] || [ -z "$password" ]; then
+  if [ -f .proxmox ]; then
+    { read username; read password; } < .proxmox
+  else
+    die 'Required: `--username` and `--password`.\n'
+  fi
+fi
+
+################################################################################
+## Getting started
+
+from_response () { echo "$response" | jq -r "$1"; }
+
+printf 'Authenticating...'
+response=$(
+    http \
+        --verify no \
+        POST $apiurl/access/ticket \
+        "username=$username" \
+        "password=$password"
+    )
+readonly csrfToken=$(from_response .data.CSRFPreventionToken)
+readonly ticket=$(from_response .data.ticket)
+printf ' done.\n'
+
+proxmox () {
+  response=$(
+    http \
+      --verify no \
+      --form \
+      "$@" \
+      "Cookie:PVEAuthCookie=$ticket" \
+      "CSRFPreventionToken:$csrfToken"
+  )
+}
+
+wait_ () {
+  upid=$1
+  while :; do
+    proxmox GET $apiurl/nodes/$node/tasks/$upid/status
+    status=$(from_response .data.status)
+    case $status in
+      running) printf '.'; sleep 1 ;;
+      stopped) break ;;
+      *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
+    esac
+  done
+}
+
+################################################################################
+## Stop VM
+
+stop_vm () {
+  printf 'Stopping VM %d...' $1
+
+  proxmox POST $apiurl/nodes/$node/qemu/$1/status/stop \
+    'overrule-shutdown'==1
+
+  wait_ $(from_response .data)
+  printf ' done.\n'
+}
+
+################################################################################
+## Delete VM
+
+delete_vm () {
+  printf 'Deleting VM %d...' $1
+
+  proxmox DELETE $apiurl/nodes/$node/qemu/$1 \
+    'destroy-unreferenced-disks'==1 \
+    'purge'==1
+
+  wait_ $(from_response .data)
+  printf ' done.\n'
+}
+
+################################################################################
+## Main loop
+
+for vmid in $vmids; do
+  stop_vm $vmid
+  delete_vm $vmid
+done

From 95389bb615d46e3d0eff6f181544eff7a3b58a6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 12:46:53 +0100
Subject: [PATCH 3/7] Remove useless piece of code

Ids must start at 100 because of Proxmox.
---
 deployment/flake-part.nix | 18 +++---------------
 deployment/procolixVm.nix | 20 ++++----------------
 2 files changed, 7 insertions(+), 31 deletions(-)

diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix
index 7d2a34f3..22158ee4 100644
--- a/deployment/flake-part.nix
+++ b/deployment/flake-part.nix
@@ -1,18 +1,6 @@
 { inputs, self, ... }:
 
 let
-  vmIdTo03d =
-    id:
-    let
-      sid = toString id;
-    in
-    if id >= 0 && id <= 9 then
-      "00${sid}"
-    else if id >= 10 && id <= 99 then
-      "0${sid}"
-    else
-      sid;
-
   allVmIds = # 100 -- 255
     let
       allVmIdsFrom = x: if x > 255 then [ ] else [ x ] ++ allVmIdsFrom (x + 1);
@@ -38,7 +26,7 @@ in
     in
     listToAttrs (
       map (vmid: {
-        name = "fedi${vmIdTo03d vmid}";
+        name = "fedi${toString vmid}";
         value = makeProvisioningConfiguration vmid;
       }) allVmIds
     );
@@ -74,8 +62,8 @@ in
         type = providers.local.exec;
         imports = [ inputs.nixops4-nixos.modules.nixops4Resource.nixos ];
         ssh.opts = "";
-        ssh.host = "95.215.187.${vmIdTo03d vmid}";
-        ssh.hostPublicKey = readFile ./hostKeys/fedi${vmIdTo03d vmid}/ssh_host_ed25519_key.pub;
+        ssh.host = "95.215.187.${toString vmid}";
+        ssh.hostPublicKey = readFile ./hostKeys/fedi${toString vmid}/ssh_host_ed25519_key.pub;
 
         nixpkgs = inputs.nixpkgs;
         nixos.module = {
diff --git a/deployment/procolixVm.nix b/deployment/procolixVm.nix
index 76efeb1d..5862de36 100644
--- a/deployment/procolixVm.nix
+++ b/deployment/procolixVm.nix
@@ -8,18 +8,6 @@
 let
   inherit (lib) mkOption;
   inherit (lib.types) types;
-
-  vmIdTo03d =
-    id:
-    let
-      sid = toString id;
-    in
-    if id >= 0 && id <= 9 then
-      "00${sid}"
-    else if id >= 10 && id <= 99 then
-      "0${sid}"
-    else
-      sid;
 in
 
 {
@@ -30,7 +18,7 @@ in
       vmid = mkOption {
         type = types.int;
         description = ''
-          Identifier of the machine. This is a number between 10 and 255.
+          Identifier of the machine. This is a number between 100 and 255.
         '';
       };
     };
@@ -43,7 +31,7 @@ in
     services.openssh.enable = true;
 
     networking = {
-      hostName = "fedi${vmIdTo03d config.procolix.vmid}";
+      hostName = "fedi${toString config.procolix.vmid}";
       domain = "procolix.com";
 
       interfaces = {
@@ -51,7 +39,7 @@ in
           ipv4 = {
             addresses = [
               {
-                address = "95.215.187.${vmIdTo03d config.procolix.vmid}";
+                address = "95.215.187.${toString config.procolix.vmid}";
                 prefixLength = 24;
               }
             ];
@@ -59,7 +47,7 @@ in
           ipv6 = {
             addresses = [
               {
-                address = "2a00:51c0:13:1305::${vmIdTo03d config.procolix.vmid}";
+                address = "2a00:51c0:13:1305::${toString config.procolix.vmid}";
                 prefixLength = 64;
               }
             ];

From 56d125a5b0142717eb83155400e53809380b911b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 13:12:06 +0100
Subject: [PATCH 4/7] Rework and cleanup provisioning script

---
 deployment/proxmox/provision.sh | 264 +++++++++++++++++++-------------
 1 file changed, 155 insertions(+), 109 deletions(-)

diff --git a/deployment/proxmox/provision.sh b/deployment/proxmox/provision.sh
index 39d74ac1..30f5f6fc 100755
--- a/deployment/proxmox/provision.sh
+++ b/deployment/proxmox/provision.sh
@@ -1,32 +1,42 @@
 #!/usr/bin/env sh
 set -euC
 
+################################################################################
+## Constants
+
+readonly apiurl=https://192.168.51.81:8006/api2/json
+
+## FIXME: There seems to be a problem with file upload where the task is
+## registered to `node051` no matter what node we are actually uploading to? For
+## now, let us just use `node051` everywhere.
+readonly node=node051
+
+readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM
+mkdir $tmpdir
+
 ################################################################################
 ## Parse arguments
 
 username=
 password=
-iso=result/iso/installer.iso
 sockets=1
 cores=1
 memory=2048
-vmid=
+vmids=
 
 help () {
   cat <<EOF
-Usage: $0 [OPTION...]
+Usage: $0 [OPTION...] [ID...]
 
-Required:
+Authentication options:
   --username STR    Username, with provider (eg. niols@pve)
   --password STR    Password
-  --vmid INT        Identifier of the VM
 
   If not provided via the command line, username and password will be looked for
   in a '.proxmox' file in the current working directory, the username on the
   first line, and the password on the second.
 
-Optional:
-  --iso PATH        Installer ISO (default: $iso)
+Other options:
   --sockets INT     Number of sockets (default: $sockets)
   --cores INT       Number of cores (default: $cores)
   --memory INT      Memory (default: $memory)
@@ -44,15 +54,16 @@ while [ $# -gt 0 ]; do
   case $argument in
     --username) readonly username=$1; shift ;;
     --password) readonly password=$1; shift ;;
-    --vmid) readonly vmid=$1; shift ;;
 
-    --iso) iso=$1; shift ;;
     --sockets) sockets=$1; shift ;;
     --cores) cores=$1; shift ;;
     --memory) memory=$1; shift ;;
 
     -h|-\?|--help) help; exit 0 ;;
-    *) die 'Unknown argument: `%s`.' "$argument" ;;
+
+    -*) die 'Unknown argument: `%s`.' "$argument" ;;
+
+    *) vmids="$vmids $argument" ;;
   esac
 done
 
@@ -64,32 +75,22 @@ if [ -z "$username" ] || [ -z "$password" ]; then
   fi
 fi
 
-[ -z "$vmid" ] && die 'Required: `--vmid`.\n'
-
-printf 'Provisioning VM %d with:\n' $vmid
-
-readonly iso
 readonly sockets
 readonly cores
 readonly memory
 
-printf '  iso: %s\n' $iso
-printf '  sockets: %d\n' $sockets
-printf '  cores: %d\n' $cores
-printf '  memory: %d\n' $memory
+## FIXME: When we figure out how to use other nodes than node051.
+# if [ -z "$node" ]; then
+#   printf 'Picking random node...'
+#   proxmox GET $apiurl/nodes
+#   node=$(from_response .data[].node | sort -R | head -n 1)
+#   printf ' done. Picked `%s`.\n' "$node"
+# fi
+# readonly node
 
 ################################################################################
 ## Getting started
 
-readonly apiurl=https://192.168.51.81:8006/api2/json
-
-## FIXME: There seems to be a problem with file upload where the task is
-## registered to `node051` no matter what node we are actually uploading to? For
-## now, let us just use `node051` everywhere.
-node=node051
-
-from_response () { echo "$response" | jq -r "$1"; }
-
 printf 'Authenticating...'
 response=$(
     http \
@@ -98,120 +99,165 @@ response=$(
         "username=$username" \
         "password=$password"
     )
-readonly csrfToken=$(from_response .data.CSRFPreventionToken)
-readonly ticket=$(from_response .data.ticket)
+readonly ticket=$(echo "$response" | jq -r .data.ticket)
+readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
 printf ' done.\n'
 
-http_ () {
-  response=$(
-    http \
-      --verify no \
-      "$@" \
-      "Cookie:PVEAuthCookie=$ticket" \
-      "CSRFPreventionToken:$csrfToken"
-  )
+proxmox () {
+  http \
+    --form \
+    --verify no \
+    "$@" \
+    "Cookie:PVEAuthCookie=$ticket" \
+    "CSRFPreventionToken:$csrfToken"
 }
 
-wait_ () {
-  upid=$1
+## Synchronous variant for when the `proxmox` function would just respond an
+## UPID in the `data` JSON field.
+proxmox_sync () {
+  response=$(proxmox "$@")
+  upid=$(echo "$response" | jq -r .data)
   while :; do
-    http_ GET $apiurl/nodes/$node/tasks/$upid/status
-    status=$(from_response .data.status)
+    response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
+    status=$(echo "$response" | jq -r .data.status)
     case $status in
-      running) printf '.'; sleep 1 ;;
+      running) sleep 1 ;;
       stopped) break ;;
-      *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
+      *) die 'unexpected status: `%s`' "$status" ;;
     esac
   done
 }
 
+################################################################################
+## Build ISO
+
+build_iso () {
+  printf 'Building ISO for VM %d...\n' $1
+
+  nix build \
+    .#isoInstallers.provisioning.fedi$1 \
+    --log-format raw --quiet \
+    --out-link $tmpdir/installer-fedi$1
+
+  ln -sf $tmpdir/installer-fedi$1/iso/installer.iso $tmpdir/installer-fedi$1.iso
+
+  printf 'done building ISO for VM %d.\n' $1
+}
+
 ################################################################################
 ## Upload ISO
 
-if [ -z "$node" ]; then
-  printf 'Picking random node...'
-  http_ GET $apiurl/nodes
-  node=$(from_response .data[].node | sort -R | head -n 1)
-  printf ' done. Picked `%s`.\n' "$node"
-fi
-readonly node
+upload_iso () {
+  printf 'Uploading ISO for VM %d...\n' $1
 
-absiso=$(cd "$(dirname "$iso")"; pwd)/$(basename "$iso")
-readonly isoname=installer-$vmid.iso
+  proxmox_sync POST $apiurl/nodes/$node/storage/local/upload \
+    filename@$tmpdir/installer-fedi$1.iso \
+    content==iso
 
-printf 'Uploading ISO...'
-ln -sf $absiso /tmp/$isoname
-http_ --form POST $apiurl/nodes/$node/storage/local/upload \
-  filename@/tmp/$isoname \
-  content==iso
-rm /tmp/$isoname
-wait_ $(from_response .data)
-printf ' done.\n'
+  printf 'done uploading ISO for VM %d.\n' $1
+}
+
+################################################################################
+## Remove ISO
+
+remove_iso () {
+  printf 'Removing ISO for VM %d... unsupported for now. (FIXME)\n' $1
+}
 
 ################################################################################
 ## Create VM
 
-printf 'Creating VM...'
+create_vm () {
+  printf 'Creating VM %d...\n' $1
 
-http_ --form POST $apiurl/nodes/$node/qemu \
-  \
-  vmid==$vmid \
-  name==$(printf 'fedi%03d' $vmid) \
-  pool==Fediversity \
-  \
-  ide2=="local:iso/$isoname,media=cdrom" \
-  ostype==l26 \
-  \
-  bios==ovmf \
-  efidisk0=='linstor_storage:1,efitype=4m' \
-  agent==1 \
-  \
-  scsihw==virtio-scsi-single \
-  scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \
-  \
-  sockets==$sockets \
-  cores==$cores \
-  cpu==x86-64-v2-AES \
-  numa==1 \
-  \
-  memory==$memory \
-  \
-  net0=='virtio,bridge=vnet1306'
+  proxmox_sync POST $apiurl/nodes/$node/qemu \
+    \
+    vmid==$1 \
+    name=="fedi$1" \
+    pool==Fediversity \
+    \
+    ide2=="local:iso/installer-fedi$1.iso,media=cdrom" \
+    ostype==l26 \
+    \
+    bios==ovmf \
+    efidisk0=='linstor_storage:1,efitype=4m' \
+    agent==1 \
+    \
+    scsihw==virtio-scsi-single \
+    scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \
+    \
+    sockets==$sockets \
+    cores==$cores \
+    cpu==x86-64-v2-AES \
+    numa==1 \
+    \
+    memory==$memory \
+    \
+    net0=='virtio,bridge=vnet1306'
 
-wait_ $(from_response .data)
-printf ' done.\n'
+  printf 'done creating VM %d.\n' $1
+}
 
 ################################################################################
 ## Install VM
 
-printf 'Installing VM...'
+install_vm () {
+  printf 'Installing VM %d...\n' $1
 
-http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
-wait_ $(from_response .data)
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
 
-while :; do
-  http_ GET $apiurl/nodes/$node/qemu/$vmid/status/current
-  status=$(from_response .data.status)
-  case $status in
-    running) printf '.'; sleep 1 ;;
-    stopped) break ;;
-    *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
-  esac
-done
+  while :; do
+    response=$(proxmox GET $apiurl/nodes/$node/qemu/$1/status/current)
+    status=$(echo "$response" | jq -r .data.status)
+    case $status in
+      running) sleep 1 ;;
+      stopped) break ;;
+      *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
+    esac
+  done
 
-printf ' done.\n'
+  printf 'done installing VM %d.\n' $1
+}
 
 ################################################################################
 ## Start VM
 
-printf 'Starting VM...'
+start_vm () {
+  printf 'Starting VM %d...\n' $1
 
-http_ --form POST $apiurl/nodes/$node/qemu/$vmid/config \
-  ide2=='none,media=cdrom' \
-  net0=='virtio,bridge=vnet1305'
-wait_ $(from_response .data)
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/config \
+    ide2=='none,media=cdrom' \
+    net0=='virtio,bridge=vnet1305'
 
-http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
-wait_ $(from_response .data)
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
 
-printf ' done.\n'
+  printf 'done starting VM %d.\n' $1
+}
+
+################################################################################
+## Main loop
+
+printf 'Provisioning VMs%s with:\n' "$vmids"
+printf '  sockets: %d\n' $sockets
+printf '  cores: %d\n' $cores
+printf '  memory: %d\n' $memory
+
+provision_vm () {
+  build_iso $1
+  upload_iso $1
+  create_vm $1
+  install_vm $1
+  start_vm $1
+  remove_iso $1
+}
+
+for vmid in $vmids; do
+  provision_vm $vmid
+done
+
+printf 'done provisioning VMs%s.\n' "$vmids"
+
+################################################################################
+## Cleanup
+
+rm -Rf $tmpdir

From 74bf29bb7504f5ba5d32008f7c4a843fd8e480dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 13:55:26 +0100
Subject: [PATCH 5/7] Parallelise provisioning script

---
 deployment/proxmox/provision.sh | 34 +++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/deployment/proxmox/provision.sh b/deployment/proxmox/provision.sh
index 30f5f6fc..6daadbae 100755
--- a/deployment/proxmox/provision.sh
+++ b/deployment/proxmox/provision.sh
@@ -46,7 +46,8 @@ Others:
 EOF
 }
 
-die () { printf "$@"; printf '\n'; help; exit 2; }
+die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
+die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; help; exit 2; }
 
 while [ $# -gt 0 ]; do
   argument=$1
@@ -61,7 +62,7 @@ while [ $# -gt 0 ]; do
 
     -h|-\?|--help) help; exit 0 ;;
 
-    -*) die 'Unknown argument: `%s`.' "$argument" ;;
+    -*) die_with_help 'Unknown argument: `%s`.' "$argument" ;;
 
     *) vmids="$vmids $argument" ;;
   esac
@@ -71,7 +72,7 @@ if [ -z "$username" ] || [ -z "$password" ]; then
   if [ -f .proxmox ]; then
     { read username; read password; } < .proxmox
   else
-    die 'Required: `--username` and `--password`.\n'
+    die_with_help 'Required: `--username` and `--password`.\n'
   fi
 fi
 
@@ -103,35 +104,48 @@ readonly ticket=$(echo "$response" | jq -r .data.ticket)
 readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
 printf ' done.\n'
 
+acquire_lock () {
+  until mkdir $tmpdir/lock-$1 2>/dev/null; do sleep 1; done
+}
+release_lock () {
+  rmdir $tmpdir/lock-$1
+}
+
 proxmox () {
+  acquire_lock proxmox
   http \
     --form \
     --verify no \
+    --ignore-stdin \
     "$@" \
     "Cookie:PVEAuthCookie=$ticket" \
     "CSRFPreventionToken:$csrfToken"
+  release_lock proxmox
 }
 
 ## Synchronous variant for when the `proxmox` function would just respond an
 ## UPID in the `data` JSON field.
-proxmox_sync () {
+proxmox_sync () (
   response=$(proxmox "$@")
   upid=$(echo "$response" | jq -r .data)
+
   while :; do
     response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
     status=$(echo "$response" | jq -r .data.status)
+
     case $status in
       running) sleep 1 ;;
       stopped) break ;;
       *) die 'unexpected status: `%s`' "$status" ;;
     esac
   done
-}
+)
 
 ################################################################################
 ## Build ISO
 
 build_iso () {
+  acquire_lock build
   printf 'Building ISO for VM %d...\n' $1
 
   nix build \
@@ -142,12 +156,14 @@ build_iso () {
   ln -sf $tmpdir/installer-fedi$1/iso/installer.iso $tmpdir/installer-fedi$1.iso
 
   printf 'done building ISO for VM %d.\n' $1
+  release_lock build
 }
 
 ################################################################################
 ## Upload ISO
 
 upload_iso () {
+  acquire_lock upload
   printf 'Uploading ISO for VM %d...\n' $1
 
   proxmox_sync POST $apiurl/nodes/$node/storage/local/upload \
@@ -155,6 +171,7 @@ upload_iso () {
     content==iso
 
   printf 'done uploading ISO for VM %d.\n' $1
+  release_lock upload
 }
 
 ################################################################################
@@ -201,7 +218,7 @@ create_vm () {
 ################################################################################
 ## Install VM
 
-install_vm () {
+install_vm () (
   printf 'Installing VM %d...\n' $1
 
   proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
@@ -217,7 +234,7 @@ install_vm () {
   done
 
   printf 'done installing VM %d.\n' $1
-}
+)
 
 ################################################################################
 ## Start VM
@@ -252,8 +269,9 @@ provision_vm () {
 }
 
 for vmid in $vmids; do
-  provision_vm $vmid
+  provision_vm $vmid &
 done
+wait
 
 printf 'done provisioning VMs%s.\n' "$vmids"
 

From 94e535688637a9dbac0463760755c2d70124cf77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 16:24:52 +0100
Subject: [PATCH 6/7] Parallelise removal script

---
 deployment/proxmox/remove.sh | 92 +++++++++++++++++++++++-------------
 1 file changed, 59 insertions(+), 33 deletions(-)

diff --git a/deployment/proxmox/remove.sh b/deployment/proxmox/remove.sh
index 259eeb12..9564fae6 100755
--- a/deployment/proxmox/remove.sh
+++ b/deployment/proxmox/remove.sh
@@ -11,6 +11,9 @@ readonly apiurl=https://192.168.51.81:8006/api2/json
 ## now, let us just use `node051` everywhere.
 readonly node=node051
 
+readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM
+mkdir $tmpdir
+
 ################################################################################
 ## Parse arguments
 
@@ -35,7 +38,8 @@ Others:
 EOF
 }
 
-die () { printf "$@"; printf '\n'; help; exit 2; }
+die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
+die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; help; exit 2; }
 
 while [ $# -gt 0 ]; do
   argument=$1
@@ -46,7 +50,7 @@ while [ $# -gt 0 ]; do
 
     -h|-\?|--help) help; exit 0 ;;
 
-    -*) die 'Unknown argument: `%s`.' "$argument" ;;
+    -*) die_with_help 'Unknown argument: `%s`.' "$argument" ;;
 
     *) vmids="$vmids $argument" ;;
   esac
@@ -56,15 +60,13 @@ if [ -z "$username" ] || [ -z "$password" ]; then
   if [ -f .proxmox ]; then
     { read username; read password; } < .proxmox
   else
-    die 'Required: `--username` and `--password`.\n'
+    die_with_help 'Required: `--username` and `--password`.\n'
   fi
 fi
 
 ################################################################################
 ## Getting started
 
-from_response () { echo "$response" | jq -r "$1"; }
-
 printf 'Authenticating...'
 response=$(
     http \
@@ -73,65 +75,89 @@ response=$(
         "username=$username" \
         "password=$password"
     )
-readonly csrfToken=$(from_response .data.CSRFPreventionToken)
-readonly ticket=$(from_response .data.ticket)
+readonly ticket=$(echo "$response" | jq -r .data.ticket)
+readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
 printf ' done.\n'
 
-proxmox () {
-  response=$(
-    http \
-      --verify no \
-      --form \
-      "$@" \
-      "Cookie:PVEAuthCookie=$ticket" \
-      "CSRFPreventionToken:$csrfToken"
-  )
+acquire_lock () {
+  until mkdir $tmpdir/lock-$1 2>/dev/null; do sleep 1; done
+}
+release_lock () {
+  rmdir $tmpdir/lock-$1
 }
 
-wait_ () {
-  upid=$1
+proxmox () {
+  acquire_lock proxmox
+  http \
+    --verify no \
+    --form \
+    "$@" \
+    "Cookie:PVEAuthCookie=$ticket" \
+    "CSRFPreventionToken:$csrfToken"
+  release_lock proxmox
+}
+
+## Synchronous variant for when the `proxmox` function would just respond an
+## UPID in the `data` JSON field.
+proxmox_sync () (
+  response=$(proxmox "$@")
+  upid=$(echo "$response" | jq -r .data)
+
   while :; do
-    proxmox GET $apiurl/nodes/$node/tasks/$upid/status
-    status=$(from_response .data.status)
+    response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
+    status=$(echo "$response" | jq -r .data.status)
+
     case $status in
-      running) printf '.'; sleep 1 ;;
+      running) sleep 1 ;;
       stopped) break ;;
-      *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
+      *) die 'unexpected status: `%s`' "$status" ;;
     esac
   done
-}
+)
 
 ################################################################################
 ## Stop VM
 
 stop_vm () {
-  printf 'Stopping VM %d...' $1
+  printf 'Stopping VM %d...\n' $1
 
-  proxmox POST $apiurl/nodes/$node/qemu/$1/status/stop \
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/stop \
     'overrule-shutdown'==1
 
-  wait_ $(from_response .data)
-  printf ' done.\n'
+  printf 'done stopping VM %d.\n' $1
 }
 
 ################################################################################
 ## Delete VM
 
 delete_vm () {
-  printf 'Deleting VM %d...' $1
+  printf 'Deleting VM %d...\n' $1
 
-  proxmox DELETE $apiurl/nodes/$node/qemu/$1 \
+  proxmox_sync DELETE $apiurl/nodes/$node/qemu/$1 \
     'destroy-unreferenced-disks'==1 \
     'purge'==1
 
-  wait_ $(from_response .data)
-  printf ' done.\n'
+  printf 'done deleting VM %d.\n' $1
 }
 
 ################################################################################
 ## Main loop
 
+printf 'Removing VMs%s...\n' "$vmids"
+
+remove_vm () {
+  stop_vm $1
+  delete_vm $1
+}
+
 for vmid in $vmids; do
-  stop_vm $vmid
-  delete_vm $vmid
+  remove_vm $vmid &
 done
+wait
+
+printf 'done removing VMs%s.\n' "$vmids"
+
+################################################################################
+## Cleanup
+
+rm -Rf $tmpdir

From 3765a7e049ce26323e036e6bd0c9624d1a3386da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20=E2=80=9CNiols=E2=80=9D=20Jeannerod?=
 <nicolas.jeannerod@moduscreate.com>
Date: Thu, 14 Nov 2024 18:14:51 +0100
Subject: [PATCH 7/7] Mention the scripts in the README

---
 deployment/proxmox/README.org | 37 ++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/deployment/proxmox/README.org b/deployment/proxmox/README.org
index 7776aa44..4163a725 100644
--- a/deployment/proxmox/README.org
+++ b/deployment/proxmox/README.org
@@ -9,6 +9,14 @@
   - Ignore “You do not have a valid subscription” message.
 * Basic terminology
 - Node :: physical host
+* Automatically
+This directory contains scripts that can automatically provision or remove a
+Proxmox VM. For now, they are tied to one node in the Fediversity Proxmox, but
+it would not be difficult to make them more generic. Try:
+#+begin_src sh
+sh provision.sh --help
+sh remove.sh --help
+#+end_src
 * Preparing the machine configuration
 - It is nicer if the machine is a QEMU guest. On NixOS:
   #+begin_src nix
@@ -23,46 +31,47 @@
   ~2a00:51c0:13:1305::XXX~.
 - Name servers should be ~95.215.185.6~ and ~95.215.185.7~.
 - Check [[https://netbox.protagio.org][Netbox]] to see which addresses are free.
-* Upload your ISO
+* Manually via the GUI
+** Upload your ISO
 - Go to Fediversity proxmox.
 - In the left view, expand under the node that you want and click on “local”.
 - Select “ISO Images”, then click “Upload”.
 - Note: You can also download from URL.
 - Note: You should click on “local” and not “local-zfs”.
-* Creating the VM
+** Creating the VM
 - Click “Create VM” at the top right corner.
-** General
+*** General
 - Node :: which node will host the VM; has to be the same
 - VM ID :: Has to be unique, probably best to use the "xxxx" in "vm0xxxx" (yet to be decided)
 - Name :: Usually "vm" + 5 digits, e.g. "vm02199"
 - Resource pool :: Fediversity
-** OS
+*** OS
 - Use CD/DVD disc image file (iso) ::
   - Storage :: local, means storage of the node.
   - ISO image :: select the image previously uploaded
 No need to touch anything else
-** System
+*** System
 - BIOS :: OVMF (UEFI)
 - EFI Storage ::  ~linstor_storage~; this is a storage shared by all of the Proxmox machines.
 - Pre-Enroll keys :: MUST be unchecked
 - Qemu Agent :: check
-** Disks
+*** Disks
 - Tick “advanced” at the bottom.
 - Disk size (GiB) :: 40 (depending on requirements)
 - SSD emulation :: check (only visible if “Advanced” is checked)
 - Discard :: check, so that blocks of removed data are cleared
-** CPU
+*** CPU
 - Sockets :: 1 (depending on requirements)
 - Cores :: 2 (depending on requirements)
 - Enable NUMA :: check
-** Memory
+*** Memory
 - Memory (MiB) :: choose what you want
 - Ballooning Device :: leave checked (only visible if “Advanced” is checked)
-** Network
+*** Network
 - Bridge :: ~vnet1306~. This is the provisioning bridge; we will change it later.
 - Firewall :: uncheck, we will handle the firewall on the VM itself
-** Confirm
-* Install and start the VM
+*** Confirm
+** Install and start the VM
 - Start the VM a first time.
   - Select the VM in the left panel. You might have to expand the node on which it is hosted.
   - Select “Console” and start the VM.
@@ -73,18 +82,18 @@ No need to touch anything else
   - Double click on the CD/DVD Drive line. Select “Do not use any media” and press OK.
   - Double click on Network Device, and change the bridge to ~vnet1305~, the public bridge.
 - Start the VM again.
-* Remove the VM
+** Remove the VM
 - [[Shutdown the VM]].
 - On the top right corner, click “More”, then “Remove”.
 - Enter the ID of the machine.
 - Check “Purge from job configurations”
 - Check “Destroy unreferenced disks owned by guest”
 - Click “Remove”.
-* Move the VM to another node
+** Move the VM to another node
 - Make sure there is no ISO plugged in.
 - Click on the VM. Click migrate. Choose target node. Go.
 - Since the storage is shared, it should go pretty fast (~1 minute).
-* Shutdown the VM
+** Shutdown the VM
 - Find the VM in the left panel.
 - At the top right corner appears a “Shutdown” button with a submenu.
 - Clicking “Shutdown” sends a signal to shutdown the machine. This might not work if the machine is not listening for that signal.