diff --git a/deployment/flake-part.nix b/deployment/flake-part.nix index 7d2a34f3..22158ee4 100644 --- a/deployment/flake-part.nix +++ b/deployment/flake-part.nix @@ -1,18 +1,6 @@ { inputs, self, ... }: let - vmIdTo03d = - id: - let - sid = toString id; - in - if id >= 0 && id <= 9 then - "00${sid}" - else if id >= 10 && id <= 99 then - "0${sid}" - else - sid; - allVmIds = # 100 -- 255 let allVmIdsFrom = x: if x > 255 then [ ] else [ x ] ++ allVmIdsFrom (x + 1); @@ -38,7 +26,7 @@ in in listToAttrs ( map (vmid: { - name = "fedi${vmIdTo03d vmid}"; + name = "fedi${toString vmid}"; value = makeProvisioningConfiguration vmid; }) allVmIds ); @@ -74,8 +62,8 @@ in type = providers.local.exec; imports = [ inputs.nixops4-nixos.modules.nixops4Resource.nixos ]; ssh.opts = ""; - ssh.host = "95.215.187.${vmIdTo03d vmid}"; - ssh.hostPublicKey = readFile ./hostKeys/fedi${vmIdTo03d vmid}/ssh_host_ed25519_key.pub; + ssh.host = "95.215.187.${toString vmid}"; + ssh.hostPublicKey = readFile ./hostKeys/fedi${toString vmid}/ssh_host_ed25519_key.pub; nixpkgs = inputs.nixpkgs; nixos.module = { diff --git a/deployment/procolixVm.nix b/deployment/procolixVm.nix index 76efeb1d..5862de36 100644 --- a/deployment/procolixVm.nix +++ b/deployment/procolixVm.nix @@ -8,18 +8,6 @@ let inherit (lib) mkOption; inherit (lib.types) types; - - vmIdTo03d = - id: - let - sid = toString id; - in - if id >= 0 && id <= 9 then - "00${sid}" - else if id >= 10 && id <= 99 then - "0${sid}" - else - sid; in { @@ -30,7 +18,7 @@ in vmid = mkOption { type = types.int; description = '' - Identifier of the machine. This is a number between 10 and 255. + Identifier of the machine. This is a number between 100 and 255. ''; }; }; @@ -43,7 +31,7 @@ in services.openssh.enable = true; networking = { - hostName = "fedi${vmIdTo03d config.procolix.vmid}"; + hostName = "fedi${toString config.procolix.vmid}"; domain = "procolix.com"; interfaces = { @@ -51,7 +39,7 @@ in ipv4 = { addresses = [ { - address = "95.215.187.${vmIdTo03d config.procolix.vmid}"; + address = "95.215.187.${toString config.procolix.vmid}"; prefixLength = 24; } ]; @@ -59,7 +47,7 @@ in ipv6 = { addresses = [ { - address = "2a00:51c0:13:1305::${vmIdTo03d config.procolix.vmid}"; + address = "2a00:51c0:13:1305::${toString config.procolix.vmid}"; prefixLength = 64; } ]; diff --git a/deployment/provision-vm.sh b/deployment/provision-vm.sh deleted file mode 100755 index c624fd79..00000000 --- a/deployment/provision-vm.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env sh -set -euC - -## Proxmox API doc: https://pve.proxmox.com/pve-docs/api-viewer - -################################################################################ -## Parse arguments - -username= -password= -iso=result/iso/installer.iso -sockets=1 -cores=1 -memory=2048 -vmid= - -help () { - cat < +#+title: Provisioning VMs via Proxmox -* Fediversity Proxmox -- http://192.168.51.81:8006/. -- It is only accessible via Procolix's VPN; see with Kevin. -- You will need identifiers. Also see with Kevin. Select “Promox VE authentication server”. -- Ignore “You do not have a valid subscription” message. +* Quick links +- Proxmox API doc :: https://pve.proxmox.com/pve-docs/api-viewer +- Fediversity Proxmox :: + - http://192.168.51.81:8006/. + - It is only accessible via Procolix's VPN; see with Kevin. + - You will need identifiers. Also see with Kevin. Select “Promox VE authentication server”. + - Ignore “You do not have a valid subscription” message. * Basic terminology - Node :: physical host +* Automatically +This directory contains scripts that can automatically provision or remove a +Proxmox VM. For now, they are tied to one node in the Fediversity Proxmox, but +it would not be difficult to make them more generic. Try: +#+begin_src sh +sh provision.sh --help +sh remove.sh --help +#+end_src * Preparing the machine configuration - It is nicer if the machine is a QEMU guest. On NixOS: #+begin_src nix @@ -23,46 +31,47 @@ ~2a00:51c0:13:1305::XXX~. - Name servers should be ~95.215.185.6~ and ~95.215.185.7~. - Check [[https://netbox.protagio.org][Netbox]] to see which addresses are free. -* Upload your ISO +* Manually via the GUI +** Upload your ISO - Go to Fediversity proxmox. - In the left view, expand under the node that you want and click on “local”. - Select “ISO Images”, then click “Upload”. - Note: You can also download from URL. - Note: You should click on “local” and not “local-zfs”. -* Creating the VM +** Creating the VM - Click “Create VM” at the top right corner. -** General +*** General - Node :: which node will host the VM; has to be the same - VM ID :: Has to be unique, probably best to use the "xxxx" in "vm0xxxx" (yet to be decided) - Name :: Usually "vm" + 5 digits, e.g. "vm02199" - Resource pool :: Fediversity -** OS +*** OS - Use CD/DVD disc image file (iso) :: - Storage :: local, means storage of the node. - ISO image :: select the image previously uploaded No need to touch anything else -** System +*** System - BIOS :: OVMF (UEFI) - EFI Storage :: ~linstor_storage~; this is a storage shared by all of the Proxmox machines. - Pre-Enroll keys :: MUST be unchecked - Qemu Agent :: check -** Disks +*** Disks - Tick “advanced” at the bottom. - Disk size (GiB) :: 40 (depending on requirements) - SSD emulation :: check (only visible if “Advanced” is checked) - Discard :: check, so that blocks of removed data are cleared -** CPU +*** CPU - Sockets :: 1 (depending on requirements) - Cores :: 2 (depending on requirements) - Enable NUMA :: check -** Memory +*** Memory - Memory (MiB) :: choose what you want - Ballooning Device :: leave checked (only visible if “Advanced” is checked) -** Network +*** Network - Bridge :: ~vnet1306~. This is the provisioning bridge; we will change it later. - Firewall :: uncheck, we will handle the firewall on the VM itself -** Confirm -* Install and start the VM +*** Confirm +** Install and start the VM - Start the VM a first time. - Select the VM in the left panel. You might have to expand the node on which it is hosted. - Select “Console” and start the VM. @@ -73,18 +82,18 @@ No need to touch anything else - Double click on the CD/DVD Drive line. Select “Do not use any media” and press OK. - Double click on Network Device, and change the bridge to ~vnet1305~, the public bridge. - Start the VM again. -* Remove the VM +** Remove the VM - [[Shutdown the VM]]. - On the top right corner, click “More”, then “Remove”. - Enter the ID of the machine. - Check “Purge from job configurations” - Check “Destroy unreferenced disks owned by guest” - Click “Remove”. -* Move the VM to another node +** Move the VM to another node - Make sure there is no ISO plugged in. - Click on the VM. Click migrate. Choose target node. Go. - Since the storage is shared, it should go pretty fast (~1 minute). -* Shutdown the VM +** Shutdown the VM - Find the VM in the left panel. - At the top right corner appears a “Shutdown” button with a submenu. - Clicking “Shutdown” sends a signal to shutdown the machine. This might not work if the machine is not listening for that signal. diff --git a/deployment/proxmox/provision.sh b/deployment/proxmox/provision.sh new file mode 100755 index 00000000..6daadbae --- /dev/null +++ b/deployment/proxmox/provision.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env sh +set -euC + +################################################################################ +## Constants + +readonly apiurl=https://192.168.51.81:8006/api2/json + +## FIXME: There seems to be a problem with file upload where the task is +## registered to `node051` no matter what node we are actually uploading to? For +## now, let us just use `node051` everywhere. +readonly node=node051 + +readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM +mkdir $tmpdir + +################################################################################ +## Parse arguments + +username= +password= +sockets=1 +cores=1 +memory=2048 +vmids= + +help () { + cat </dev/null; do sleep 1; done +} +release_lock () { + rmdir $tmpdir/lock-$1 +} + +proxmox () { + acquire_lock proxmox + http \ + --form \ + --verify no \ + --ignore-stdin \ + "$@" \ + "Cookie:PVEAuthCookie=$ticket" \ + "CSRFPreventionToken:$csrfToken" + release_lock proxmox +} + +## Synchronous variant for when the `proxmox` function would just respond an +## UPID in the `data` JSON field. +proxmox_sync () ( + response=$(proxmox "$@") + upid=$(echo "$response" | jq -r .data) + + while :; do + response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status) + status=$(echo "$response" | jq -r .data.status) + + case $status in + running) sleep 1 ;; + stopped) break ;; + *) die 'unexpected status: `%s`' "$status" ;; + esac + done +) + +################################################################################ +## Build ISO + +build_iso () { + acquire_lock build + printf 'Building ISO for VM %d...\n' $1 + + nix build \ + .#isoInstallers.provisioning.fedi$1 \ + --log-format raw --quiet \ + --out-link $tmpdir/installer-fedi$1 + + ln -sf $tmpdir/installer-fedi$1/iso/installer.iso $tmpdir/installer-fedi$1.iso + + printf 'done building ISO for VM %d.\n' $1 + release_lock build +} + +################################################################################ +## Upload ISO + +upload_iso () { + acquire_lock upload + printf 'Uploading ISO for VM %d...\n' $1 + + proxmox_sync POST $apiurl/nodes/$node/storage/local/upload \ + filename@$tmpdir/installer-fedi$1.iso \ + content==iso + + printf 'done uploading ISO for VM %d.\n' $1 + release_lock upload +} + +################################################################################ +## Remove ISO + +remove_iso () { + printf 'Removing ISO for VM %d... unsupported for now. (FIXME)\n' $1 +} + +################################################################################ +## Create VM + +create_vm () { + printf 'Creating VM %d...\n' $1 + + proxmox_sync POST $apiurl/nodes/$node/qemu \ + \ + vmid==$1 \ + name=="fedi$1" \ + pool==Fediversity \ + \ + ide2=="local:iso/installer-fedi$1.iso,media=cdrom" \ + ostype==l26 \ + \ + bios==ovmf \ + efidisk0=='linstor_storage:1,efitype=4m' \ + agent==1 \ + \ + scsihw==virtio-scsi-single \ + scsi0=='linstor_storage:32,discard=on,ssd=on,iothread=on' \ + \ + sockets==$sockets \ + cores==$cores \ + cpu==x86-64-v2-AES \ + numa==1 \ + \ + memory==$memory \ + \ + net0=='virtio,bridge=vnet1306' + + printf 'done creating VM %d.\n' $1 +} + +################################################################################ +## Install VM + +install_vm () ( + printf 'Installing VM %d...\n' $1 + + proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start + + while :; do + response=$(proxmox GET $apiurl/nodes/$node/qemu/$1/status/current) + status=$(echo "$response" | jq -r .data.status) + case $status in + running) sleep 1 ;; + stopped) break ;; + *) printf ' unexpected status: `%s`\n' "$status"; exit 2 ;; + esac + done + + printf 'done installing VM %d.\n' $1 +) + +################################################################################ +## Start VM + +start_vm () { + printf 'Starting VM %d...\n' $1 + + proxmox_sync POST $apiurl/nodes/$node/qemu/$1/config \ + ide2=='none,media=cdrom' \ + net0=='virtio,bridge=vnet1305' + + proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/start + + printf 'done starting VM %d.\n' $1 +} + +################################################################################ +## Main loop + +printf 'Provisioning VMs%s with:\n' "$vmids" +printf ' sockets: %d\n' $sockets +printf ' cores: %d\n' $cores +printf ' memory: %d\n' $memory + +provision_vm () { + build_iso $1 + upload_iso $1 + create_vm $1 + install_vm $1 + start_vm $1 + remove_iso $1 +} + +for vmid in $vmids; do + provision_vm $vmid & +done +wait + +printf 'done provisioning VMs%s.\n' "$vmids" + +################################################################################ +## Cleanup + +rm -Rf $tmpdir diff --git a/deployment/proxmox/remove.sh b/deployment/proxmox/remove.sh new file mode 100755 index 00000000..9564fae6 --- /dev/null +++ b/deployment/proxmox/remove.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env sh +set -euC + +################################################################################ +## Constants + +readonly apiurl=https://192.168.51.81:8006/api2/json + +## FIXME: There seems to be a problem with file upload where the task is +## registered to `node051` no matter what node we are actually uploading to? For +## now, let us just use `node051` everywhere. +readonly node=node051 + +readonly tmpdir=/tmp/proxmox-provision-$RANDOM$RANDOM +mkdir $tmpdir + +################################################################################ +## Parse arguments + +username= +password= +vmids= + +help () { + cat </dev/null; do sleep 1; done +} +release_lock () { + rmdir $tmpdir/lock-$1 +} + +proxmox () { + acquire_lock proxmox + http \ + --verify no \ + --form \ + "$@" \ + "Cookie:PVEAuthCookie=$ticket" \ + "CSRFPreventionToken:$csrfToken" + release_lock proxmox +} + +## Synchronous variant for when the `proxmox` function would just respond an +## UPID in the `data` JSON field. +proxmox_sync () ( + response=$(proxmox "$@") + upid=$(echo "$response" | jq -r .data) + + while :; do + response=$(proxmox GET $apiurl/nodes/$node/tasks/$upid/status) + status=$(echo "$response" | jq -r .data.status) + + case $status in + running) sleep 1 ;; + stopped) break ;; + *) die 'unexpected status: `%s`' "$status" ;; + esac + done +) + +################################################################################ +## Stop VM + +stop_vm () { + printf 'Stopping VM %d...\n' $1 + + proxmox_sync POST $apiurl/nodes/$node/qemu/$1/status/stop \ + 'overrule-shutdown'==1 + + printf 'done stopping VM %d.\n' $1 +} + +################################################################################ +## Delete VM + +delete_vm () { + printf 'Deleting VM %d...\n' $1 + + proxmox_sync DELETE $apiurl/nodes/$node/qemu/$1 \ + 'destroy-unreferenced-disks'==1 \ + 'purge'==1 + + printf 'done deleting VM %d.\n' $1 +} + +################################################################################ +## Main loop + +printf 'Removing VMs%s...\n' "$vmids" + +remove_vm () { + stop_vm $1 + delete_vm $1 +} + +for vmid in $vmids; do + remove_vm $vmid & +done +wait + +printf 'done removing VMs%s.\n' "$vmids" + +################################################################################ +## Cleanup + +rm -Rf $tmpdir