Improve automated provisioning/removal of Proxmox VMs (#6)

Reviewed-on: Fediversity/Fediversity#6
Reviewed-by: Valentin Gagarin <valentin.gagarin@tweag.io>

deployment/flake-part.nix

@@ -1,18 +1,6 @@
 { inputs, self, ... }:
 
 let
-  vmIdTo03d =
-    id:
-    let
-      sid = toString id;
-    in
-    if id >= 0 && id <= 9 then
-      "00${sid}"
-    else if id >= 10 && id <= 99 then
-      "0${sid}"
-    else
-      sid;
-
   allVmIds = # 100 -- 255
     let
       allVmIdsFrom = x: if x > 255 then [ ] else [ x ] ++ allVmIdsFrom (x + 1);
@@ -38,7 +26,7 @@ in
     in
     listToAttrs (
       map (vmid: {
-        name = "fedi${vmIdTo03d vmid}";
+        name = "fedi${toString vmid}";
         value = makeProvisioningConfiguration vmid;
       }) allVmIds
     );
@@ -74,8 +62,8 @@ in
         type = providers.local.exec;
         imports = [ inputs.nixops4-nixos.modules.nixops4Resource.nixos ];
         ssh.opts = "";
-        ssh.host = "95.215.187.${vmIdTo03d vmid}";
+        ssh.host = "95.215.187.${toString vmid}";
-        ssh.hostPublicKey = readFile ./hostKeys/fedi${vmIdTo03d vmid}/ssh_host_ed25519_key.pub;
+        ssh.hostPublicKey = readFile ./hostKeys/fedi${toString vmid}/ssh_host_ed25519_key.pub;
 
         nixpkgs = inputs.nixpkgs;
         nixos.module = {

deployment/procolixVm.nix

@@ -8,18 +8,6 @@
 let
   inherit (lib) mkOption;
   inherit (lib.types) types;
-
-  vmIdTo03d =
-    id:
-    let
-      sid = toString id;
-    in
-    if id >= 0 && id <= 9 then
-      "00${sid}"
-    else if id >= 10 && id <= 99 then
-      "0${sid}"
-    else
-      sid;
 in
 
 {
@@ -30,7 +18,7 @@ in
       vmid = mkOption {
         type = types.int;
         description = ''
-          Identifier of the machine. This is a number between 10 and 255.
+          Identifier of the machine. This is a number between 100 and 255.
         '';
       };
     };
@@ -43,7 +31,7 @@ in
     services.openssh.enable = true;
 
     networking = {
-      hostName = "fedi${vmIdTo03d config.procolix.vmid}";
+      hostName = "fedi${toString config.procolix.vmid}";
       domain = "procolix.com";
 
       interfaces = {
@@ -51,7 +39,7 @@ in
           ipv4 = {
             addresses = [
               {
-                address = "95.215.187.${vmIdTo03d config.procolix.vmid}";
+                address = "95.215.187.${toString config.procolix.vmid}";
                 prefixLength = 24;
               }
             ];
@@ -59,7 +47,7 @@ in
           ipv6 = {
             addresses = [
               {
-                address = "2a00:51c0:13:1305::${vmIdTo03d config.procolix.vmid}";
+                address = "2a00:51c0:13:1305::${toString config.procolix.vmid}";
                 prefixLength = 64;
               }
             ];

deployment/provision-vm.sh

@@ -1,223 +0,0 @@
-#!/usr/bin/env sh
-set -euC
-
-## Proxmox API doc: https://pve.proxmox.com/pve-docs/api-viewer
-
-################################################################################
-## Parse arguments
-
-username=
-password=
-iso=result/iso/installer.iso
-sockets=1
-cores=1
-memory=2048
-vmid=
-
-help () {
-  cat <<EOF
-Usage: $0 [OPTION...]
-
-Required:
-  --username STR    Username, with provider (eg. niols@pve)
-  --password STR    Password
-  --vmid INT        Identifier of the VM
-
-  If not provided via the command line, username and password will be looked for
-  in a `.proxmox` file in the current working directory, the username on the
-  first line, and the password on the second.
-
-Optional:
-  --iso PATH        Installer ISO (default: $iso)
-  --sockets INT     Number of sockets (default: $sockets)
-  --cores INT       Number of cores (default: $cores)
-  --memory INT      Memory (default: $memory)
-
-Others:
-  -h|-?|--help      Show this help and exit
-EOF
-}
-
-die () { printf "$@"; printf '\n'; help; exit 2; }
-
-while [ $# -gt 0 ]; do
-  argument=$1
-  shift
-  case $argument in
-    --username) readonly username=$1; shift ;;
-    --password) readonly password=$1; shift ;;
-    --vmid) readonly vmid=$1; shift ;;
-
-    --iso) iso=$1; shift ;;
-    --sockets) sockets=$1; shift ;;
-    --cores) cores=$1; shift ;;
-    --memory) memory=$1; shift ;;
-
-    -h|-\?|--help) help; exit 0 ;;
-    *) die 'Unknown argument: `%s`.' "$argument" ;;
-  esac
-done
-
-if [ -z "$username" ] || [ -z "$password" ]; then
-  if [ -f .proxmox ]; then
-    { read username; read password; } < .proxmox
-  else
-    die 'Required: `--username` and `--password`.\n'
-  fi
-fi
-
-[ -z "$vmid" ] && die 'Required: `--vmid`.\n'
-
-printf 'Configuration:\n'
-
-printf '  username: %s\n' $username
-printf '  password: %s\n' $password
-printf '  vmid: %s\n' $vmid
-
-readonly iso
-readonly sockets
-readonly cores
-readonly memory
-
-printf '  iso: %s\n' $iso
-printf '  sockets: %d\n' $sockets
-printf '  cores: %d\n' $cores
-printf '  memory: %d\n' $memory
-
-################################################################################
-## Getting started
-
-readonly apiurl=https://192.168.51.81:8006/api2/json
-
-## FIXME: There seems to be a problem with file upload where the task is
-## registered to `node051` no matter what node we are actually uploading to? For
-## now, let us just use `node051` everywhere.
-node=node051
-
-from_response () { echo "$response" | jq -r "$1"; }
-
-printf 'Authenticating...'
-response=$(
-    http \
-        --verify no \
-        POST $apiurl/access/ticket \
-        "username=$username" \
-        "password=$password"
-    )
-readonly csrfToken=$(from_response .data.CSRFPreventionToken)
-readonly ticket=$(from_response .data.ticket)
-printf ' done.\n'
-
-http_ () {
-  response=$(
-    http \
-      --verify no \
-      "$@" \
-      "Cookie:PVEAuthCookie=$ticket" \
-      "CSRFPreventionToken:$csrfToken"
-  )
-}
-
-wait_ () {
-  upid=$1
-  while :; do
-    http_ GET $apiurl/nodes/$node/tasks/$upid/status
-    status=$(from_response .data.status)
-    case $status in
-      running) printf '.'; sleep 1 ;;
-      stopped) break ;;
-      *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
-    esac
-  done
-}
-
-################################################################################
-## Upload ISO
-
-if [ -z "$node" ]; then
-  printf 'Picking random node...'
-  http_ GET $apiurl/nodes
-  node=$(from_response .data[].node | sort -R | head -n 1)
-  printf ' done. Picked `%s`.\n' "$node"
-fi
-readonly node
-
-absiso=$(cd "$(dirname "$iso")"; pwd)/$(basename "$iso")
-readonly isoname=installer-$vmid.iso
-
-printf 'Uploading ISO...'
-ln -sf $absiso /tmp/$isoname
-http_ --form POST $apiurl/nodes/$node/storage/local/upload \
-  filename@/tmp/$isoname \
-  content==iso
-rm /tmp/$isoname
-wait_ $(from_response .data)
-printf ' done.\n'
-
-################################################################################
-## Create VM
-
-printf 'Creating VM...'
-
-http_ --form POST $apiurl/nodes/$node/qemu \
-  \
-  vmid==$vmid \
-  name==$(printf 'fedi%03d' $vmid) \
-  pool==Fediversity \
-  \
-  ide2=="local:iso/$isoname,media=cdrom" \
-  ostype==l26 \
-  \
-  bios==ovmf \
-  efidisk0=='linstor_storage:1,efitype=4m' \
-  agent==1 \
-  \
-  scsihw==virtio-scsi-single \
-  scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \
-  \
-  sockets==$sockets \
-  cores==$cores \
-  cpu==x86-64-v2-AES \
-  numa==1 \
-  \
-  memory==$memory \
-  \
-  net0=='virtio,bridge=vnet1306'
-
-wait_ $(from_response .data)
-printf ' done.\n'
-
-################################################################################
-## Install VM
-
-printf 'Installing VM...'
-
-http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
-wait_ $(from_response .data)
-
-while :; do
-  http_ GET $apiurl/nodes/$node/qemu/$vmid/status/current
-  status=$(from_response .data.status)
-  case $status in
-    running) printf '.'; sleep 1 ;;
-    stopped) break ;;
-    *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
-  esac
-done
-
-printf 'done.\n'
-
-################################################################################
-## Start VM
-
-printf 'Starting VM...'
-
-http_ --form POST $apiurl/nodes/$node/qemu/$vmid/config \
-  ide2=='none,media=cdrom' \
-  net0=='virtio,bridge=vnet1305'
-wait_ $(from_response .data)
-
-http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
-wait_ $(from_response .data)
-
-printf 'done.\n'

deployment/proxmox/README.org

@@ -1,14 +1,22 @@
-#+title: Provisioning a Proxmox VM
+#+title: Provisioning VMs via Proxmox
-#+author: Kevin Muller, Hans van Zijst & Nicolas Jeannerod
-#+date: <2024-10-25 Fri>
 
-* Fediversity Proxmox
+* Quick links
+- Proxmox API doc :: https://pve.proxmox.com/pve-docs/api-viewer
+- Fediversity Proxmox ::
   - http://192.168.51.81:8006/.
   - It is only accessible via Procolix's VPN; see with Kevin.
   - You will need identifiers. Also see with Kevin. Select “Promox VE authentication server”.
   - Ignore “You do not have a valid subscription” message.
 * Basic terminology
 - Node :: physical host
+* Automatically
+This directory contains scripts that can automatically provision or remove a
+Proxmox VM. For now, they are tied to one node in the Fediversity Proxmox, but
+it would not be difficult to make them more generic. Try:
+#+begin_src sh
+sh provision.sh --help
+sh remove.sh --help
+#+end_src
 * Preparing the machine configuration
 - It is nicer if the machine is a QEMU guest. On NixOS:
   #+begin_src nix
@@ -23,46 +31,47 @@
  ~2a00:51c0:13:1305::XXX~.
 - Name servers should be ~95.215.185.6~ and ~95.215.185.7~.
 - Check [[https://netbox.protagio.org][Netbox]] to see which addresses are free.
-* Upload your ISO
+* Manually via the GUI
+** Upload your ISO
 - Go to Fediversity proxmox.
 - In the left view, expand under the node that you want and click on “local”.
 - Select “ISO Images”, then click “Upload”.
 - Note: You can also download from URL.
 - Note: You should click on “local” and not “local-zfs”.
-* Creating the VM
+** Creating the VM
 - Click “Create VM” at the top right corner.
-** General
+*** General
 - Node :: which node will host the VM; has to be the same
 - VM ID :: Has to be unique, probably best to use the "xxxx" in "vm0xxxx" (yet to be decided)
 - Name :: Usually "vm" + 5 digits, e.g. "vm02199"
 - Resource pool :: Fediversity
-** OS
+*** OS
 - Use CD/DVD disc image file (iso) ::
   - Storage :: local, means storage of the node.
   - ISO image :: select the image previously uploaded
 No need to touch anything else
-** System
+*** System
 - BIOS :: OVMF (UEFI)
 - EFI Storage :: ~linstor_storage~; this is a storage shared by all of the Proxmox machines.
 - Pre-Enroll keys :: MUST be unchecked
 - Qemu Agent :: check
-** Disks
+*** Disks
 - Tick “advanced” at the bottom.
 - Disk size (GiB) :: 40 (depending on requirements)
 - SSD emulation :: check (only visible if “Advanced” is checked)
 - Discard :: check, so that blocks of removed data are cleared
-** CPU
+*** CPU
 - Sockets :: 1 (depending on requirements)
 - Cores :: 2 (depending on requirements)
 - Enable NUMA :: check
-** Memory
+*** Memory
 - Memory (MiB) :: choose what you want
 - Ballooning Device :: leave checked (only visible if “Advanced” is checked)
-** Network
+*** Network
 - Bridge :: ~vnet1306~. This is the provisioning bridge; we will change it later.
 - Firewall :: uncheck, we will handle the firewall on the VM itself
-** Confirm
+*** Confirm
-* Install and start the VM
+** Install and start the VM
 - Start the VM a first time.
   - Select the VM in the left panel. You might have to expand the node on which it is hosted.
   - Select “Console” and start the VM.
@@ -73,18 +82,18 @@ No need to touch anything else
   - Double click on the CD/DVD Drive line. Select “Do not use any media” and press OK.
   - Double click on Network Device, and change the bridge to ~vnet1305~, the public bridge.
 - Start the VM again.
-* Remove the VM
+** Remove the VM
 - [[Shutdown the VM]].
 - On the top right corner, click “More”, then “Remove”.
 - Enter the ID of the machine.
 - Check “Purge from job configurations”
 - Check “Destroy unreferenced disks owned by guest”
 - Click “Remove”.
-* Move the VM to another node
+** Move the VM to another node
 - Make sure there is no ISO plugged in.
 - Click on the VM. Click migrate. Choose target node. Go.
 - Since the storage is shared, it should go pretty fast (~1 minute).
-* Shutdown the VM
+** Shutdown the VM
 - Find the VM in the left panel.
 - At the top right corner appears a “Shutdown” button with a submenu.
 - Clicking “Shutdown” sends a signal to shutdown the machine. This might not work if the machine is not listening for that signal.

deployment/proxmox/provision.sh

@@ -0,0 +1,281 @@
+#!/usr/bin/env sh
+set -euC
+
+################################################################################
+## Constants
+
+readonly apiurl=https://192.168.51.81:8006/api2/json
+
+## FIXME: There seems to be a problem with file upload where the task is
+## registered to `node051` no matter what node we are actually uploading to? For
+## now, let us just use `node051` everywhere.
+readonly node=node051
+
+readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM
+mkdir $tmpdir
+
+################################################################################
+## Parse arguments
+
+username=
+password=
+sockets=1
+cores=1
+memory=2048
+vmids=
+
+help () {
+  cat <<EOF
+Usage: $0 [OPTION...] [ID...]
+
+Authentication options:
+  --username STR    Username, with provider (eg. niols@pve)
+  --password STR    Password
+
+  If not provided via the command line, username and password will be looked for
+  in a '.proxmox' file in the current working directory, the username on the
+  first line, and the password on the second.
+
+Other options:
+  --sockets INT     Number of sockets (default: $sockets)
+  --cores INT       Number of cores (default: $cores)
+  --memory INT      Memory (default: $memory)
+
+Others:
+  -h|-?|--help      Show this help and exit
+EOF
+}
+
+die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
+die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; help; exit 2; }
+
+while [ $# -gt 0 ]; do
+  argument=$1
+  shift
+  case $argument in
+    --username) readonly username=$1; shift ;;
+    --password) readonly password=$1; shift ;;
+
+    --sockets) sockets=$1; shift ;;
+    --cores) cores=$1; shift ;;
+    --memory) memory=$1; shift ;;
+
+    -h|-\?|--help) help; exit 0 ;;
+
+    -*) die_with_help 'Unknown argument: `%s`.' "$argument" ;;
+
+    *) vmids="$vmids $argument" ;;
+  esac
+done
+
+if [ -z "$username" ] || [ -z "$password" ]; then
+  if [ -f .proxmox ]; then
+    { read username; read password; } < .proxmox
+  else
+    die_with_help 'Required: `--username` and `--password`.\n'
+  fi
+fi
+
+readonly sockets
+readonly cores
+readonly memory
+
+## FIXME: When we figure out how to use other nodes than node051.
+# if [ -z "$node" ]; then
+#   printf 'Picking random node...'
+#   proxmox GET $apiurl/nodes
+#   node=$(from_response .data[].node | sort -R | head -n 1)
+#   printf ' done. Picked `%s`.\n' "$node"
+# fi
+# readonly node
+
+################################################################################
+## Getting started
+
+printf 'Authenticating...'
+response=$(
+    http \
+        --verify no \
+        POST $apiurl/access/ticket \
+        "username=$username" \
+        "password=$password"
+    )
+readonly ticket=$(echo "$response" | jq -r .data.ticket)
+readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
+printf ' done.\n'
+
+acquire_lock () {
+  until mkdir $tmpdir/lock-$1 2>/dev/null; do sleep 1; done
+}
+release_lock () {
+  rmdir $tmpdir/lock-$1
+}
+
+proxmox () {
+  acquire_lock proxmox
+  http \
+    --form \
+    --verify no \
+    --ignore-stdin \
+    "$@" \
+    "Cookie:PVEAuthCookie=$ticket" \
+    "CSRFPreventionToken:$csrfToken"
+  release_lock proxmox
+}
+
+## Synchronous variant for when the `proxmox` function would just respond an
+## UPID in the `data` JSON field.
+proxmox_sync () (
+  response=$(proxmox "$@")
+  upid=$(echo "$response" | jq -r .data)
+
+  while :; do
+    response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
+    status=$(echo "$response" | jq -r .data.status)
+
+    case $status in
+      running) sleep 1 ;;
+      stopped) break ;;
+      *) die 'unexpected status: `%s`' "$status" ;;
+    esac
+  done
+)
+
+################################################################################
+## Build ISO
+
+build_iso () {
+  acquire_lock build
+  printf 'Building ISO for VM %d...\n' $1
+
+  nix build \
+    .#isoInstallers.provisioning.fedi$1 \
+    --log-format raw --quiet \
+    --out-link $tmpdir/installer-fedi$1
+
+  ln -sf $tmpdir/installer-fedi$1/iso/installer.iso $tmpdir/installer-fedi$1.iso
+
+  printf 'done building ISO for VM %d.\n' $1
+  release_lock build
+}
+
+################################################################################
+## Upload ISO
+
+upload_iso () {
+  acquire_lock upload
+  printf 'Uploading ISO for VM %d...\n' $1
+
+  proxmox_sync POST $apiurl/nodes/$node/storage/local/upload \
+    filename@$tmpdir/installer-fedi$1.iso \
+    content==iso
+
+  printf 'done uploading ISO for VM %d.\n' $1
+  release_lock upload
+}
+
+################################################################################
+## Remove ISO
+
+remove_iso () {
+  printf 'Removing ISO for VM %d... unsupported for now. (FIXME)\n' $1
+}
+
+################################################################################
+## Create VM
+
+create_vm () {
+  printf 'Creating VM %d...\n' $1
+
+  proxmox_sync POST $apiurl/nodes/$node/qemu \
+    \
+    vmid==$1 \
+    name=="fedi$1" \
+    pool==Fediversity \
+    \
+    ide2=="local:iso/installer-fedi$1.iso,media=cdrom" \
+    ostype==l26 \
+    \
+    bios==ovmf \
+    efidisk0=='linstor_storage:1,efitype=4m' \
+    agent==1 \
+    \
+    scsihw==virtio-scsi-single \
+    scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \
+    \
+    sockets==$sockets \
+    cores==$cores \
+    cpu==x86-64-v2-AES \
+    numa==1 \
+    \
+    memory==$memory \
+    \
+    net0=='virtio,bridge=vnet1306'
+
+  printf 'done creating VM %d.\n' $1
+}
+
+################################################################################
+## Install VM
+
+install_vm () (
+  printf 'Installing VM %d...\n' $1
+
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
+
+  while :; do
+    response=$(proxmox GET $apiurl/nodes/$node/qemu/$1/status/current)
+    status=$(echo "$response" | jq -r .data.status)
+    case $status in
+      running) sleep 1 ;;
+      stopped) break ;;
+      *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
+    esac
+  done
+
+  printf 'done installing VM %d.\n' $1
+)
+
+################################################################################
+## Start VM
+
+start_vm () {
+  printf 'Starting VM %d...\n' $1
+
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/config \
+    ide2=='none,media=cdrom' \
+    net0=='virtio,bridge=vnet1305'
+
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
+
+  printf 'done starting VM %d.\n' $1
+}
+
+################################################################################
+## Main loop
+
+printf 'Provisioning VMs%s with:\n' "$vmids"
+printf '  sockets: %d\n' $sockets
+printf '  cores: %d\n' $cores
+printf '  memory: %d\n' $memory
+
+provision_vm () {
+  build_iso $1
+  upload_iso $1
+  create_vm $1
+  install_vm $1
+  start_vm $1
+  remove_iso $1
+}
+
+for vmid in $vmids; do
+  provision_vm $vmid &
+done
+wait
+
+printf 'done provisioning VMs%s.\n' "$vmids"
+
+################################################################################
+## Cleanup
+
+rm -Rf $tmpdir

deployment/proxmox/remove.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env sh
+set -euC
+
+################################################################################
+## Constants
+
+readonly apiurl=https://192.168.51.81:8006/api2/json
+
+## FIXME: There seems to be a problem with file upload where the task is
+## registered to `node051` no matter what node we are actually uploading to? For
+## now, let us just use `node051` everywhere.
+readonly node=node051
+
+readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM
+mkdir $tmpdir
+
+################################################################################
+## Parse arguments
+
+username=
+password=
+vmids=
+
+help () {
+  cat <<EOF
+Usage: $0 [OPTION...] [ID...]
+
+Authentication options:
+  --username STR    Username, with provider (eg. niols@pve)
+  --password STR    Password
+
+  If not provided via the command line, username and password will be looked for
+  in a '.proxmox' file in the current working directory, the username on the
+  first line, and the password on the second.
+
+Others:
+  -h|-?|--help      Show this help and exit
+EOF
+}
+
+die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
+die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; help; exit 2; }
+
+while [ $# -gt 0 ]; do
+  argument=$1
+  shift
+  case $argument in
+    --username) readonly username=$1; shift ;;
+    --password) readonly password=$1; shift ;;
+
+    -h|-\?|--help) help; exit 0 ;;
+
+    -*) die_with_help 'Unknown argument: `%s`.' "$argument" ;;
+
+    *) vmids="$vmids $argument" ;;
+  esac
+done
+
+if [ -z "$username" ] || [ -z "$password" ]; then
+  if [ -f .proxmox ]; then
+    { read username; read password; } < .proxmox
+  else
+    die_with_help 'Required: `--username` and `--password`.\n'
+  fi
+fi
+
+################################################################################
+## Getting started
+
+printf 'Authenticating...'
+response=$(
+    http \
+        --verify no \
+        POST $apiurl/access/ticket \
+        "username=$username" \
+        "password=$password"
+    )
+readonly ticket=$(echo "$response" | jq -r .data.ticket)
+readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
+printf ' done.\n'
+
+acquire_lock () {
+  until mkdir $tmpdir/lock-$1 2>/dev/null; do sleep 1; done
+}
+release_lock () {
+  rmdir $tmpdir/lock-$1
+}
+
+proxmox () {
+  acquire_lock proxmox
+  http \
+    --verify no \
+    --form \
+    "$@" \
+    "Cookie:PVEAuthCookie=$ticket" \
+    "CSRFPreventionToken:$csrfToken"
+  release_lock proxmox
+}
+
+## Synchronous variant for when the `proxmox` function would just respond an
+## UPID in the `data` JSON field.
+proxmox_sync () (
+  response=$(proxmox "$@")
+  upid=$(echo "$response" | jq -r .data)
+
+  while :; do
+    response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
+    status=$(echo "$response" | jq -r .data.status)
+
+    case $status in
+      running) sleep 1 ;;
+      stopped) break ;;
+      *) die 'unexpected status: `%s`' "$status" ;;
+    esac
+  done
+)
+
+################################################################################
+## Stop VM
+
+stop_vm () {
+  printf 'Stopping VM %d...\n' $1
+
+  proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/stop \
+    'overrule-shutdown'==1
+
+  printf 'done stopping VM %d.\n' $1
+}
+
+################################################################################
+## Delete VM
+
+delete_vm () {
+  printf 'Deleting VM %d...\n' $1
+
+  proxmox_sync DELETE $apiurl/nodes/$node/qemu/$1 \
+    'destroy-unreferenced-disks'==1 \
+    'purge'==1
+
+  printf 'done deleting VM %d.\n' $1
+}
+
+################################################################################
+## Main loop
+
+printf 'Removing VMs%s...\n' "$vmids"
+
+remove_vm () {
+  stop_vm $1
+  delete_vm $1
+}
+
+for vmid in $vmids; do
+  remove_vm $vmid &
+done
+wait
+
+printf 'done removing VMs%s.\n' "$vmids"
+
+################################################################################
+## Cleanup
+
+rm -Rf $tmpdir