Fediversity/infra/proxmox-provision.sh

#!/usr/bin/env bash
set -euC

################################################################################
## Constants

## FIXME: There seems to be a problem with file upload where the task is
## registered to `node051` no matter what node we are actually uploading to? For
## now, let us just use `node051` everywhere.
node=node051

readonly tmpdir=/tmp/proxmox-provision-$RANDOM
mkdir $tmpdir

################################################################################
## Parse arguments

api_url=
username=
password=
vm_names=

debug=false

help () {
  cat <<EOF
Usage: $0 [OPTION...] NAME [NAME...]

NAME is a string identifying the VM in the flake. This script will look for a
'vmOptions.<NAME>' and 'nixosConfigurations.<NAME>' to get the informations that
it needs.

Options:
  --api-url STR     Base URL of the Proxmox API (required)
  --username STR    Username, with provider (eg. niols@pve; required)
  --password STR    Password (required)

  --debug           Run this script in debug mode (default: $debug)
  -h|-?|--help      Show this help and exit

Options can also be provided by adding assignments to a '.proxmox' file in the
current working directory. For instance, it could contain:

  api_url=https://192.168.51.81:8006/api2/json
  username=mireille@pve
  debug=true

Command line options take precedence over options found in the '.proxmox' file.
EOF
}

# shellcheck disable=SC2059
die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
# shellcheck disable=SC2059
die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n\n'; help; exit 2; }

# shellcheck disable=SC2059
debug () { if $debug; then printf >&2 '\033[37m'; printf >&2 "$@"; printf >&2 '\033[0m\n'; fi }

if [ -f .proxmox ]; then
  # shellcheck disable=SC1091
  . "$PWD"/.proxmox
fi

while [ $# -gt 0 ]; do
  argument=$1
  shift
  case $argument in
    --api-url|--api_url) readonly api_url="$1"; shift ;;
    --username) readonly username="$1"; shift ;;
    --password) readonly password="$1"; shift ;;
    --node) readonly node="$1"; shift ;;

    --debug) debug=true ;;

    -h|-\?|--help) help; exit 0 ;;

    -*) die_with_help "Unknown argument: '%s'." "$argument" ;;

    *) vm_names="$vm_names $argument" ;;
  esac
done

if [ -z "$vm_names" ]; then
  die_with_help "Required: at least one VM name."
fi

if [ -z "$api_url" ] || [ -z "$username" ] || [ -z "$password" ]; then
  die_with_help "Required: '--api-url', '--username' and '--password'."
fi

## FIXME: When we figure out how to use other nodes than node051.
# if [ -z "$node" ]; then
#   printf 'Picking random node...'
#   proxmox GET "$api_url/nodes"
#   node=$(from_response .data[].node | sort -R | head -n 1)
#   printf " done. Picked '%s'.\n" "$node"
# fi
# readonly node

readonly debug

################################################################################
## Getting started

printf 'Authenticating...'
response=$(
    http \
        --verify no \
        POST "$api_url/access/ticket" \
        "username=$username" \
        "password=$password"
    )
ticket=$(echo "$response" | jq -r .data.ticket)
readonly ticket
csrf_token=$(echo "$response" | jq -r .data.CSRFPreventionToken)
readonly csrf_token
printf ' done.\n'

acquire_lock () {
  until mkdir "$tmpdir/lock-$1" 2>/dev/null; do sleep 1; done
}
release_lock () {
  rmdir "$tmpdir/lock-$1"
}

proxmox () {
  acquire_lock proxmox
  debug 'request %s' "$*"
  response=$(
    http \
      --form \
      --verify no \
      --ignore-stdin \
      "$@" \
      "Cookie:PVEAuthCookie=$ticket" \
      "CSRFPreventionToken:$csrf_token"
  )
  debug 'response to request %s:\n  %s' "$*" "$response"
  release_lock proxmox
  echo "$response"
}

## Synchronous variant for when the `proxmox` function would just respond an
## UPID in the `data` JSON field.
proxmox_sync () (
  response=$(proxmox "$@")
  upid=$(echo "$response" | jq -r .data)

  while :; do
    response=$(proxmox GET "$api_url/nodes/$node/tasks/$upid/status")
    status=$(echo "$response" | jq -r .data.status)

    case $status in
      running) sleep 1 ;;
      stopped) break ;;
      *) die "unexpected status: '%s'" "$status" ;;
    esac
  done
)

################################################################################
## Grab VM options
##
## Takes the name of the VM, grabs `.#vmOptions.<name>` and defines a bunch of
## global variables corresponding to all the options.

grab_vm_options () {
  local options

  vm_name=$1

  printf 'Grabing VM options for VM %s...\n' "$vm_name"

  options=$(
    # nix --extra-experimental-features 'nix-command flakes' eval \
    #   --impure --raw --expr "
    #     builtins.toJSON (builtins.getFlake (builtins.toString ./.)).vmOptions.$vm_name
    #   " \
    # --log-format raw --quiet
    echo '
      {
        "description":"",
        "sockets":1,
        "cores":1,
        "memory":2048,
        "diskSize":32,
        "name":"test14",
        "vmId":7014,
        "hostPublicKey":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHTbxDzq3xFeLvrXs6tyTE08o3CekYZmqFeGmkcHmf21",
        "unsafeHostPrivateKey":"-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\nQyNTUxOQAAACB028Q86t8RXi7617OrckxNPKNwnpGGZqhXhppHB5n9tQAAAIhfhYlCX4WJ\nQgAAAAtzc2gtZWQyNTUxOQAAACB028Q86t8RXi7617OrckxNPKNwnpGGZqhXhppHB5n9tQ\nAAAEAualLRodpovSzGAhza2OVvg5Yp8xv3A7xUNNbKsMTKSHTbxDzq3xFeLvrXs6tyTE08\no3CekYZmqFeGmkcHmf21AAAAAAECAwQF\n-----END OPENSSH PRIVATE KEY-----\n"
      }
    '
  )

  vm_id=$(echo "$options" | jq -r .vmId)
  description=$(echo "$options" | jq -r .description)

  sockets=$(echo "$options" | jq -r .sockets)
  cores=$(echo "$options" | jq -r .cores)
  memory=$(echo "$options" | jq -r .memory)
  disk_size=$(echo "$options" | jq -r .diskSize)

  host_public_key=$(echo "$options" | jq -r .hostPublicKey)
  host_private_key=$(echo "$options" | jq -r .unsafeHostPrivateKey)

  if [ "$host_private_key" != null ] && [ "$host_public_key" = null ]; then
    die 'I do not know what to do with a private key but no public key.'
  fi

  printf 'done grabing VM options for VM %s. Got:\n  id: %d\n  sockets: %d\n  cores: %d\n  memory: %d MiB\n  disk size: %d GiB\n' \
    "$vm_name" "$vm_id" "$sockets" "$cores" "$memory" "$disk_size"
}

################################################################################
## Build ISO

build_iso () {
  local nix_host_keys

  acquire_lock build
  printf 'Building ISO for VM %s...\n' "$vm_name"

  if [ "$host_private_key" != null ]; then
    echo "$host_public_key" > "$tmpdir"/"$vm_name"_host_key.pub
    echo "$host_private_key" > "$tmpdir"/"$vm_name"_host_key
    nix_host_keys="
      hostKeys.ed25519 = {
        public = $tmpdir/${vm_name}_host_key.pub;
        private = $tmpdir/${vm_name}_host_key;
      };
    "
  else
    nix_host_keys=
  fi

  nix --extra-experimental-features 'nix-command flakes' build \
  #   --impure --expr "
  #     let flake = builtins.getFlake (builtins.toString ./.); in
  #     import ./infra/makeInstallerIso.nix {
  #       nixosConfiguration = flake.nixosConfigurations.$vm_name;
  #       # FIXME pass nixpkgs from npins
  #       $nix_host_keys
  #     }
  #   " \
  #   --log-format raw --quiet \
  #   --out-link "$tmpdir/installer-$vm_name"

  # nix --extra-experimental-features 'nix-command' build \
  #   --impure --expr "
  #     import ./infra/makeInstallerIso.nix {
  #       # nixosConfiguration = $configuration;
  #       nixosConfiguration = import $configuration;
  #       $nix_host_keys
  #     }
  #   " \
  #   --log-format raw --quiet \
  #   --out-link "$tmpdir/installer-$vm_name"

  # TODO after install: $nix_host_keys
  # cp $tmpdir/${vm_name}_host_key /mnt/etc/ssh/ssh_host_ed25519_key
  # chmod 600 /mnt/etc/ssh/ssh_host_ed25519_key
  # cp $tmpdir/${vm_name}_host_key.pub /mnt/etc/ssh/ssh_host_ed25519_key.pub
  # chmod 644 /mnt/etc/ssh/ssh_host_ed25519_key.pub

  # nix --extra-experimental-features 'nix-command' build \
    --impure --expr "
      (import $configuration).config.system.build.image
    " \
    --log-format raw --quiet \
    --out-link "$tmpdir/installer-$vm_name"

  # ls "$tmpdir/installer-$vm_name"
  # ls "$tmpdir/installer-$vm_name/image.raw"

  # shellcheck disable=SC2181
  if [ $? -ne 0 ]; then
    die 'Something went wrong when building ISO for VM %s.
Check the Nix logs and fix things. Possibly there just is no NixOS configuration by that name?' \
      "$vm_name"
  fi

  # ln -sf "$(ls "$tmpdir/installer-$vm_name"/iso/nixos-*.iso)" "$tmpdir/installer-$vm_name.iso"
  ln -sf "$(ls "$tmpdir/installer-$vm_name"/image.raw)" "$tmpdir/installer-$vm_name.raw"

  printf 'done building ISO for VM %s.\n' "$vm_name"
  release_lock build
}

################################################################################
## Upload ISO

upload_iso () {
  acquire_lock upload
  printf 'Uploading ISO for VM %s...\n' "$vm_name"

  proxmox_sync POST "$api_url/nodes/$node/storage/local/upload" \
    "filename@$tmpdir/installer-$vm_name.raw" \
    content==raw

  printf 'done uploading ISO for VM %s.\n' "$vm_name"
  release_lock upload
}

################################################################################
## Remove ISO

remove_iso () {
  printf 'Removing ISO for VM %s...\n' "$vm_name"

  proxmox_sync DELETE "$api_url/nodes/$node/storage/local/content/local:iso/installer-$vm_name.raw"

  printf 'done removing ISO for VM %s.\n' "$vm_name"
}

################################################################################
## Create VM

create_vm () {
  printf 'Creating VM %s...\n' "$vm_name"

  proxmox_sync POST "$api_url/nodes/$node/qemu" \
    \
    vmid=="$vm_id" \
    name=="$vm_name" \
    pool==Fediversity \
    description=="$description" \
    \
    ide2=="local:iso/installer-$vm_name.raw,media=cdrom" \
    ostype==l26 \
    \
    bios==ovmf \
    efidisk0=='linstor_storage:1,efitype=4m' \
    agent==1 \
    \
    scsihw==virtio-scsi-single \
    scsi0=="linstor_storage:$disk_size,discard=on,ssd=on,iothread=on" \
    \
    sockets=="$sockets" \
    cores=="$cores" \
    cpu==x86-64-v2-AES \
    numa==1 \
    \
    memory=="$memory" \
    \
    net0=='virtio,bridge=vnet1306'

  printf 'done creating VM %s.\n' "$vm_name"
}

################################################################################
## Install VM

install_vm () (
  printf 'Installing VM %s...\n' "$vm_name"

  proxmox_sync POST "$api_url/nodes/$node/qemu/$vm_id/status/start"

  while :; do
    response=$(proxmox GET "$api_url/nodes/$node/qemu/$vm_id/status/current")
    status=$(echo "$response" | jq -r .data.status)
    case $status in
      running) sleep 1 ;;
      stopped) break ;;
      *) die " unexpected status: '%s'\n" "$status" ;;
    esac
  done

  printf 'done installing VM %s.\n' "$vm_name"
)

################################################################################
## Start VM

start_vm () {
  printf 'Starting VM %s...\n' "$vm_name"

  proxmox_sync POST "$api_url/nodes/$node/qemu/$vm_id/config" \
    ide2=='none,media=cdrom' \
    net0=='virtio,bridge=vnet1305'

  proxmox_sync POST "$api_url/nodes/$node/qemu/$vm_id/status/start"

  printf 'done starting VM %s.\n' "$vm_name"
}

################################################################################
## Main loop

printf 'Provisioning VMs%s...\n' "$vm_names"

provision_vm () (
  ## Grab VM options and put them in global variables. NOTE: Mind the fact that
  ## we now run in a sub-shell, allowing us to define global variables without
  ## clashing with concurrent executions of `provision_vm`.
  grab_vm_options "$1"
  build_iso
  upload_iso
  create_vm
  install_vm
  start_vm
  remove_iso
)

# FIXME make vm_names a thing from $vm_name to $configuration?
# for vm_name in $vm_names; do
#   provision_vm "$vm_name" &
# done
for chunk in $vm_names; do
  IFS=: read -r vm_name configuration <<< "$chunk"
  provision_vm "$vm_name" "$configuration" &
done

nb_errors=0
while :; do
  wait -n && :
  case $? in
    0) ;;
    127) break ;;
    *) nb_errors=$((nb_errors + 1)) ;;
  esac
done
if [ "$nb_errors" != 0 ]; then
  die 'encountered %d errors while provisioning VMs%s.' "$nb_errors" "$vm_names"
fi


printf 'done provisioning VMs%s.\n' "$vm_names"

################################################################################
## Cleanup

rm -Rf $tmpdir
exit 0