fricklerhandwerk/Fediversity
1
0
Fork 0
forked from Fediversity/Fediversity
Code Pull requests Activity

Improve automated provisioning/removal of Proxmox VMs (#6)

Browse source 
Reviewed-on: Fediversity/Fediversity#6
Reviewed-by: Valentin Gagarin <valentin.gagarin@tweag.io>
This commit is contained in:
Nicolas Jeannerod 2024-11-17 00:09:21 +01:00
parent 07fa942989 3765a7e049
commit bd478eb32b
6 changed files with 482 additions and 276 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
deployment/flake-part.nix
View file

@ -1,18 +1,6 @@
{ inputs, self, ... }:
let
vmIdTo03d =
id:
let
sid = toString id;
in
if id >= 0 && id <= 9 then
"00${sid}"
else if id >= 10 && id <= 99 then
"0${sid}"
else
sid;
allVmIds = # 100 -- 255
let
allVmIdsFrom = x: if x > 255 then [ ] else [ x ] ++ allVmIdsFrom (x + 1);
@ -38,7 +26,7 @@ in
in
listToAttrs (
map (vmid: {
name = "fedi${vmIdTo03d vmid}";
name = "fedi${toString vmid}";
value = makeProvisioningConfiguration vmid;
}) allVmIds
);
@ -74,8 +62,8 @@ in
type = providers.local.exec;
imports = [ inputs.nixops4-nixos.modules.nixops4Resource.nixos ];
ssh.opts = "";
ssh.host = "95.215.187.${vmIdTo03d vmid}";
ssh.hostPublicKey = readFile ./hostKeys/fedi${vmIdTo03d vmid}/ssh_host_ed25519_key.pub;
ssh.host = "95.215.187.${toString vmid}";
ssh.hostPublicKey = readFile ./hostKeys/fedi${toString vmid}/ssh_host_ed25519_key.pub;
nixpkgs = inputs.nixpkgs;
nixos.module = {

20
deployment/procolixVm.nix
View file

@ -8,18 +8,6 @@
let
inherit (lib) mkOption;
inherit (lib.types) types;
vmIdTo03d =
id:
let
sid = toString id;
in
if id >= 0 && id <= 9 then
"00${sid}"
else if id >= 10 && id <= 99 then
"0${sid}"
else
sid;
in
{
@ -30,7 +18,7 @@ in
vmid = mkOption {
type = types.int;
description = ''
Identifier of the machine. This is a number between 10 and 255.
Identifier of the machine. This is a number between 100 and 255.
'';
};
};
@ -43,7 +31,7 @@ in
services.openssh.enable = true;
networking = {
hostName = "fedi${vmIdTo03d config.procolix.vmid}";
hostName = "fedi${toString config.procolix.vmid}";
domain = "procolix.com";
interfaces = {
@ -51,7 +39,7 @@ in
ipv4 = {
addresses = [
{
address = "95.215.187.${vmIdTo03d config.procolix.vmid}";
address = "95.215.187.${toString config.procolix.vmid}";
prefixLength = 24;
}
];
@ -59,7 +47,7 @@ in
ipv6 = {
addresses = [
{
address = "2a00:51c0:13:1305::${vmIdTo03d config.procolix.vmid}";
address = "2a00:51c0:13:1305::${toString config.procolix.vmid}";
prefixLength = 64;
}
];

223
deployment/provision-vm.sh
View file

@ -1,223 +0,0 @@
#!/usr/bin/env sh
set -euC
## Proxmox API doc: https://pve.proxmox.com/pve-docs/api-viewer
################################################################################
## Parse arguments
username=
password=
iso=result/iso/installer.iso
sockets=1
cores=1
memory=2048
vmid=
help () {
cat <<EOF
Usage: $0 [OPTION...]
Required:
--username STR Username, with provider (eg. niols@pve)
--password STR Password
--vmid INT Identifier of the VM
If not provided via the command line, username and password will be looked for
in a `.proxmox` file in the current working directory, the username on the
first line, and the password on the second.
Optional:
--iso PATH Installer ISO (default: $iso)
--sockets INT Number of sockets (default: $sockets)
--cores INT Number of cores (default: $cores)
--memory INT Memory (default: $memory)
Others:
-h|-?|--help Show this help and exit
EOF
}
die () { printf "$@"; printf '\n'; help; exit 2; }
while [ $# -gt 0 ]; do
argument=$1
shift
case $argument in
--username) readonly username=$1; shift ;;
--password) readonly password=$1; shift ;;
--vmid) readonly vmid=$1; shift ;;
--iso) iso=$1; shift ;;
--sockets) sockets=$1; shift ;;
--cores) cores=$1; shift ;;
--memory) memory=$1; shift ;;
-h|-\?|--help) help; exit 0 ;;
*) die 'Unknown argument: `%s`.' "$argument" ;;
esac
done
if [ -z "$username" ] || [ -z "$password" ]; then
if [ -f .proxmox ]; then
{ read username; read password; } < .proxmox
else
die 'Required: `--username` and `--password`.\n'
fi
fi
[ -z "$vmid" ] && die 'Required: `--vmid`.\n'
printf 'Configuration:\n'
printf ' username: %s\n' $username
printf ' password: %s\n' $password
printf ' vmid: %s\n' $vmid
readonly iso
readonly sockets
readonly cores
readonly memory
printf ' iso: %s\n' $iso
printf ' sockets: %d\n' $sockets
printf ' cores: %d\n' $cores
printf ' memory: %d\n' $memory
################################################################################
## Getting started
readonly apiurl=https://192.168.51.81:8006/api2/json
## FIXME: There seems to be a problem with file upload where the task is
## registered to `node051` no matter what node we are actually uploading to? For
## now, let us just use `node051` everywhere.
node=node051
from_response () { echo "$response" | jq -r "$1"; }
printf 'Authenticating...'
response=$(
http \
--verify no \
POST $apiurl/access/ticket \
"username=$username" \
"password=$password"
)
readonly csrfToken=$(from_response .data.CSRFPreventionToken)
readonly ticket=$(from_response .data.ticket)
printf ' done.\n'
http_ () {
response=$(
http \
--verify no \
"$@" \
"Cookie:PVEAuthCookie=$ticket" \
"CSRFPreventionToken:$csrfToken"
)
}
wait_ () {
upid=$1
while :; do
http_ GET $apiurl/nodes/$node/tasks/$upid/status
status=$(from_response .data.status)
case $status in
running) printf '.'; sleep 1 ;;
stopped) break ;;
*) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
esac
done
}
################################################################################
## Upload ISO
if [ -z "$node" ]; then
printf 'Picking random node...'
http_ GET $apiurl/nodes
node=$(from_response .data[].node | sort -R | head -n 1)
printf ' done. Picked `%s`.\n' "$node"
fi
readonly node
absiso=$(cd "$(dirname "$iso")"; pwd)/$(basename "$iso")
readonly isoname=installer-$vmid.iso
printf 'Uploading ISO...'
ln -sf $absiso /tmp/$isoname
http_ --form POST $apiurl/nodes/$node/storage/local/upload \
filename@/tmp/$isoname \
content==iso
rm /tmp/$isoname
wait_ $(from_response .data)
printf ' done.\n'
################################################################################
## Create VM
printf 'Creating VM...'
http_ --form POST $apiurl/nodes/$node/qemu \
\
vmid==$vmid \
name==$(printf 'fedi%03d' $vmid) \
pool==Fediversity \
\
ide2=="local:iso/$isoname,media=cdrom" \
ostype==l26 \
\
bios==ovmf \
efidisk0=='linstor_storage:1,efitype=4m' \
agent==1 \
\
scsihw==virtio-scsi-single \
scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \
\
sockets==$sockets \
cores==$cores \
cpu==x86-64-v2-AES \
numa==1 \
\
memory==$memory \
\
net0=='virtio,bridge=vnet1306'
wait_ $(from_response .data)
printf ' done.\n'
################################################################################
## Install VM
printf 'Installing VM...'
http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
wait_ $(from_response .data)
while :; do
http_ GET $apiurl/nodes/$node/qemu/$vmid/status/current
status=$(from_response .data.status)
case $status in
running) printf '.'; sleep 1 ;;
stopped) break ;;
*) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
esac
done
printf 'done.\n'
################################################################################
## Start VM
printf 'Starting VM...'
http_ --form POST $apiurl/nodes/$node/qemu/$vmid/config \
ide2=='none,media=cdrom' \
net0=='virtio,bridge=vnet1305'
wait_ $(from_response .data)
http_ POST $apiurl/nodes/$node/qemu/$vmid/status/start
wait_ $(from_response .data)
printf 'done.\n'

45
proxmox/proxmox-provisioning.org → deployment/proxmox/README.org
View file

@ -1,14 +1,22 @@
#+title: Provisioning a Proxmox VM
#+author: Kevin Muller, Hans van Zijst & Nicolas Jeannerod
#+date: <2024-10-25 Fri>
#+title: Provisioning VMs via Proxmox
* Fediversity Proxmox
* Quick links
- Proxmox API doc :: https://pve.proxmox.com/pve-docs/api-viewer
- Fediversity Proxmox ::
- http://192.168.51.81:8006/.
- It is only accessible via Procolix's VPN; see with Kevin.
- You will need identifiers. Also see with Kevin. Select “Promox VE authentication server”.
- Ignore “You do not have a valid subscription” message.
* Basic terminology
- Node :: physical host
* Automatically
This directory contains scripts that can automatically provision or remove a
Proxmox VM. For now, they are tied to one node in the Fediversity Proxmox, but
it would not be difficult to make them more generic. Try:
#+begin_src sh
sh provision.sh --help
sh remove.sh --help
#+end_src
* Preparing the machine configuration
- It is nicer if the machine is a QEMU guest. On NixOS:
#+begin_src nix
@ -23,46 +31,47 @@
~2a00:51c0:13:1305::XXX~.
- Name servers should be ~95.215.185.6~ and ~95.215.185.7~.
- Check [[https://netbox.protagio.org][Netbox]] to see which addresses are free.
* Upload your ISO
* Manually via the GUI
** Upload your ISO
- Go to Fediversity proxmox.
- In the left view, expand under the node that you want and click on “local”.
- Select “ISO Images”, then click “Upload”.
- Note: You can also download from URL.
- Note: You should click on “local” and not “local-zfs”.
* Creating the VM
** Creating the VM
- Click “Create VM” at the top right corner.
** General
*** General
- Node :: which node will host the VM; has to be the same
- VM ID :: Has to be unique, probably best to use the "xxxx" in "vm0xxxx" (yet to be decided)
- Name :: Usually "vm" + 5 digits, e.g. "vm02199"
- Resource pool :: Fediversity
** OS
*** OS
- Use CD/DVD disc image file (iso) ::
- Storage :: local, means storage of the node.
- ISO image :: select the image previously uploaded
No need to touch anything else
** System
*** System
- BIOS :: OVMF (UEFI)
- EFI Storage :: ~linstor_storage~; this is a storage shared by all of the Proxmox machines.
- Pre-Enroll keys :: MUST be unchecked
- Qemu Agent :: check
** Disks
*** Disks
- Tick “advanced” at the bottom.
- Disk size (GiB) :: 40 (depending on requirements)
- SSD emulation :: check (only visible if “Advanced” is checked)
- Discard :: check, so that blocks of removed data are cleared
** CPU
*** CPU
- Sockets :: 1 (depending on requirements)
- Cores :: 2 (depending on requirements)
- Enable NUMA :: check
** Memory
*** Memory
- Memory (MiB) :: choose what you want
- Ballooning Device :: leave checked (only visible if “Advanced” is checked)
** Network
*** Network
- Bridge :: ~vnet1306~. This is the provisioning bridge; we will change it later.
- Firewall :: uncheck, we will handle the firewall on the VM itself
** Confirm
* Install and start the VM
*** Confirm
** Install and start the VM
- Start the VM a first time.
- Select the VM in the left panel. You might have to expand the node on which it is hosted.
- Select “Console” and start the VM.
@ -73,18 +82,18 @@ No need to touch anything else
- Double click on the CD/DVD Drive line. Select “Do not use any media” and press OK.
- Double click on Network Device, and change the bridge to ~vnet1305~, the public bridge.
- Start the VM again.
* Remove the VM
** Remove the VM
- [[Shutdown the VM]].
- On the top right corner, click “More”, then “Remove”.
- Enter the ID of the machine.
- Check “Purge from job configurations”
- Check “Destroy unreferenced disks owned by guest”
- Click “Remove”.
* Move the VM to another node
** Move the VM to another node
- Make sure there is no ISO plugged in.
- Click on the VM. Click migrate. Choose target node. Go.
- Since the storage is shared, it should go pretty fast (~1 minute).
* Shutdown the VM
** Shutdown the VM
- Find the VM in the left panel.
- At the top right corner appears a “Shutdown” button with a submenu.
- Clicking “Shutdown” sends a signal to shutdown the machine. This might not work if the machine is not listening for that signal.

281
deployment/proxmox/provision.sh Executable file
View file

@ -0,0 +1,281 @@
#!/usr/bin/env sh
set -euC
################################################################################
## Constants
readonly apiurl=https://192.168.51.81:8006/api2/json
## FIXME: There seems to be a problem with file upload where the task is
## registered to `node051` no matter what node we are actually uploading to? For
## now, let us just use `node051` everywhere.
readonly node=node051
readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM
mkdir $tmpdir
################################################################################
## Parse arguments
username=
password=
sockets=1
cores=1
memory=2048
vmids=
help () {
cat <<EOF
Usage: $0 [OPTION...] [ID...]
Authentication options:
--username STR Username, with provider (eg. niols@pve)
--password STR Password
If not provided via the command line, username and password will be looked for
in a '.proxmox' file in the current working directory, the username on the
first line, and the password on the second.
Other options:
--sockets INT Number of sockets (default: $sockets)
--cores INT Number of cores (default: $cores)
--memory INT Memory (default: $memory)
Others:
-h|-?|--help Show this help and exit
EOF
}
die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; help; exit 2; }
while [ $# -gt 0 ]; do
argument=$1
shift
case $argument in
--username) readonly username=$1; shift ;;
--password) readonly password=$1; shift ;;
--sockets) sockets=$1; shift ;;
--cores) cores=$1; shift ;;
--memory) memory=$1; shift ;;
-h|-\?|--help) help; exit 0 ;;
-*) die_with_help 'Unknown argument: `%s`.' "$argument" ;;
*) vmids="$vmids $argument" ;;
esac
done
if [ -z "$username" ] || [ -z "$password" ]; then
if [ -f .proxmox ]; then
{ read username; read password; } < .proxmox
else
die_with_help 'Required: `--username` and `--password`.\n'
fi
fi
readonly sockets
readonly cores
readonly memory
## FIXME: When we figure out how to use other nodes than node051.
# if [ -z "$node" ]; then
# printf 'Picking random node...'
# proxmox GET $apiurl/nodes
# node=$(from_response .data[].node | sort -R | head -n 1)
# printf ' done. Picked `%s`.\n' "$node"
# fi
# readonly node
################################################################################
## Getting started
printf 'Authenticating...'
response=$(
http \
--verify no \
POST $apiurl/access/ticket \
"username=$username" \
"password=$password"
)
readonly ticket=$(echo "$response" | jq -r .data.ticket)
readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
printf ' done.\n'
acquire_lock () {
until mkdir $tmpdir/lock-$1 2>/dev/null; do sleep 1; done
}
release_lock () {
rmdir $tmpdir/lock-$1
}
proxmox () {
acquire_lock proxmox
http \
--form \
--verify no \
--ignore-stdin \
"$@" \
"Cookie:PVEAuthCookie=$ticket" \
"CSRFPreventionToken:$csrfToken"
release_lock proxmox
}
## Synchronous variant for when the `proxmox` function would just respond an
## UPID in the `data` JSON field.
proxmox_sync () (
response=$(proxmox "$@")
upid=$(echo "$response" | jq -r .data)
while :; do
response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
status=$(echo "$response" | jq -r .data.status)
case $status in
running) sleep 1 ;;
stopped) break ;;
*) die 'unexpected status: `%s`' "$status" ;;
esac
done
)
################################################################################
## Build ISO
build_iso () {
acquire_lock build
printf 'Building ISO for VM %d...\n' $1
nix build \
.#isoInstallers.provisioning.fedi$1 \
--log-format raw --quiet \
--out-link $tmpdir/installer-fedi$1
ln -sf $tmpdir/installer-fedi$1/iso/installer.iso $tmpdir/installer-fedi$1.iso
printf 'done building ISO for VM %d.\n' $1
release_lock build
}
################################################################################
## Upload ISO
upload_iso () {
acquire_lock upload
printf 'Uploading ISO for VM %d...\n' $1
proxmox_sync POST $apiurl/nodes/$node/storage/local/upload \
filename@$tmpdir/installer-fedi$1.iso \
content==iso
printf 'done uploading ISO for VM %d.\n' $1
release_lock upload
}
################################################################################
## Remove ISO
remove_iso () {
printf 'Removing ISO for VM %d... unsupported for now. (FIXME)\n' $1
}
################################################################################
## Create VM
create_vm () {
printf 'Creating VM %d...\n' $1
proxmox_sync POST $apiurl/nodes/$node/qemu \
\
vmid==$1 \
name=="fedi$1" \
pool==Fediversity \
\
ide2=="local:iso/installer-fedi$1.iso,media=cdrom" \
ostype==l26 \
\
bios==ovmf \
efidisk0=='linstor_storage:1,efitype=4m' \
agent==1 \
\
scsihw==virtio-scsi-single \
scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \
\
sockets==$sockets \
cores==$cores \
cpu==x86-64-v2-AES \
numa==1 \
\
memory==$memory \
\
net0=='virtio,bridge=vnet1306'
printf 'done creating VM %d.\n' $1
}
################################################################################
## Install VM
install_vm () (
printf 'Installing VM %d...\n' $1
proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
while :; do
response=$(proxmox GET $apiurl/nodes/$node/qemu/$1/status/current)
status=$(echo "$response" | jq -r .data.status)
case $status in
running) sleep 1 ;;
stopped) break ;;
*) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;;
esac
done
printf 'done installing VM %d.\n' $1
)
################################################################################
## Start VM
start_vm () {
printf 'Starting VM %d...\n' $1
proxmox_sync POST $apiurl/nodes/$node/qemu/$1/config \
ide2=='none,media=cdrom' \
net0=='virtio,bridge=vnet1305'
proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start
printf 'done starting VM %d.\n' $1
}
################################################################################
## Main loop
printf 'Provisioning VMs%s with:\n' "$vmids"
printf ' sockets: %d\n' $sockets
printf ' cores: %d\n' $cores
printf ' memory: %d\n' $memory
provision_vm () {
build_iso $1
upload_iso $1
create_vm $1
install_vm $1
start_vm $1
remove_iso $1
}
for vmid in $vmids; do
provision_vm $vmid &
done
wait
printf 'done provisioning VMs%s.\n' "$vmids"
################################################################################
## Cleanup
rm -Rf $tmpdir

163
deployment/proxmox/remove.sh Executable file
View file

@ -0,0 +1,163 @@
#!/usr/bin/env sh
set -euC
################################################################################
## Constants
readonly apiurl=https://192.168.51.81:8006/api2/json
## FIXME: There seems to be a problem with file upload where the task is
## registered to `node051` no matter what node we are actually uploading to? For
## now, let us just use `node051` everywhere.
readonly node=node051
readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM
mkdir $tmpdir
################################################################################
## Parse arguments
username=
password=
vmids=
help () {
cat <<EOF
Usage: $0 [OPTION...] [ID...]
Authentication options:
--username STR Username, with provider (eg. niols@pve)
--password STR Password
If not provided via the command line, username and password will be looked for
in a '.proxmox' file in the current working directory, the username on the
first line, and the password on the second.
Others:
-h|-?|--help Show this help and exit
EOF
}
die () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; exit 2; }
die_with_help () { printf '\033[31m'; printf "$@"; printf '\033[0m\n'; help; exit 2; }
while [ $# -gt 0 ]; do
argument=$1
shift
case $argument in
--username) readonly username=$1; shift ;;
--password) readonly password=$1; shift ;;
-h|-\?|--help) help; exit 0 ;;
-*) die_with_help 'Unknown argument: `%s`.' "$argument" ;;
*) vmids="$vmids $argument" ;;
esac
done
if [ -z "$username" ] || [ -z "$password" ]; then
if [ -f .proxmox ]; then
{ read username; read password; } < .proxmox
else
die_with_help 'Required: `--username` and `--password`.\n'
fi
fi
################################################################################
## Getting started
printf 'Authenticating...'
response=$(
http \
--verify no \
POST $apiurl/access/ticket \
"username=$username" \
"password=$password"
)
readonly ticket=$(echo "$response" | jq -r .data.ticket)
readonly csrfToken=$(echo "$response" | jq -r .data.CSRFPreventionToken)
printf ' done.\n'
acquire_lock () {
until mkdir $tmpdir/lock-$1 2>/dev/null; do sleep 1; done
}
release_lock () {
rmdir $tmpdir/lock-$1
}
proxmox () {
acquire_lock proxmox
http \
--verify no \
--form \
"$@" \
"Cookie:PVEAuthCookie=$ticket" \
"CSRFPreventionToken:$csrfToken"
release_lock proxmox
}
## Synchronous variant for when the `proxmox` function would just respond an
## UPID in the `data` JSON field.
proxmox_sync () (
response=$(proxmox "$@")
upid=$(echo "$response" | jq -r .data)
while :; do
response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status)
status=$(echo "$response" | jq -r .data.status)
case $status in
running) sleep 1 ;;
stopped) break ;;
*) die 'unexpected status: `%s`' "$status" ;;
esac
done
)
################################################################################
## Stop VM
stop_vm () {
printf 'Stopping VM %d...\n' $1
proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/stop \
'overrule-shutdown'==1
printf 'done stopping VM %d.\n' $1
}
################################################################################
## Delete VM
delete_vm () {
printf 'Deleting VM %d...\n' $1
proxmox_sync DELETE $apiurl/nodes/$node/qemu/$1 \
'destroy-unreferenced-disks'==1 \
'purge'==1
printf 'done deleting VM %d.\n' $1
}
################################################################################
## Main loop
printf 'Removing VMs%s...\n' "$vmids"
remove_vm () {
stop_vm $1
delete_vm $1
}
for vmid in $vmids; do
remove_vm $vmid &
done
wait
printf 'done removing VMs%s.\n' "$vmids"
################################################################################
## Cleanup
rm -Rf $tmpdir