pr-script-test: build vm/ scripts, and every ct/ script on Incus too

- vm/ scripts run unattended (VM_UNATTENDED=1) on the Proxmox hosts, IDs
  9100-9199, one VM per host at a time. Checks: success line, the VM keeps
  running, a DHCP lease, and a ${VM_IP} URL answering 200. Console
  screenshot from QEMU when there is no web UI. ARM64-only scripts are
  listed as not built.
- Every ct/ script now builds on pve and incus (new incus-test runners,
  incus-ci-host-setup.sh); the comment gets a Platform column.
- Spinner frames are dropped from logs and live output.
- The sweeper ignores and removes locks of killed jobs (older than 3h).

.github/workflows/scripts/incus-ci-host-setup.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# One-time setup of an Incus test host behind pr-script-test.yml: the same
+# ct/ scripts as on Proxmox, built through community-scripts/core's Incus
+# backend. Safe to re-run. Run as root on a disposable Debian 13 machine (it
+# builds pull request code as root).
+#
+#   incus-ci-host-setup.sh                       host only
+#   incus-ci-host-setup.sh <owner/repo> <token>  host + a runner for that repo
+#
+# The token is a runner *registration* token (valid one hour):
+#   gh api -X POST repos/<owner/repo>/actions/runners/registration-token --jq .token
+
+set -euo pipefail
+
+RUNNER_VERSION="${RUNNER_VERSION:-2.338.2}"
+SERVE_ROOT=/srv/ci
+SERVE_PORT=18080
+NET_ADDR=10.98.0.1
+NET_CIDR=10.98.0.0/24
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq incus jq curl python3 iptables chromium fonts-dejavu fonts-noto-color-emoji >/dev/null
+
+# Containers get a NATed bridge of their own (Incus runs DHCP and DNS on it)
+# and a plain directory pool on the root disk.
+if ! incus storage show default &>/dev/null; then
+  incus admin init --preseed <<EOF
+networks:
+- name: incusbr0
+  type: bridge
+  config:
+    ipv4.address: ${NET_ADDR}/24
+    ipv4.nat: "true"
+    ipv6.address: none
+storage_pools:
+- name: default
+  driver: dir
+profiles:
+- name: default
+  devices:
+    root:
+      path: /
+      pool: default
+      type: disk
+    eth0:
+      name: eth0
+      network: incusbr0
+      type: nic
+EOF
+fi
+
+# Screenshots of the app under test are taken as this user, never as root.
+id ci-shot &>/dev/null || useradd --system --create-home --shell /usr/sbin/nologin ci-shot
+
+# Same rules as on the Proxmox hosts: the screenshot browser may reach the
+# test containers, DNS and the public internet, nothing private, and not this
+# host's own address on the container network (the PR tree server).
+cat >/usr/local/sbin/ci-shot-firewall <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+uid=\$(id -u ci-shot)
+for fw in iptables ip6tables; do
+  \$fw -w -N CI_SHOT 2>/dev/null || \$fw -w -F CI_SHOT
+  \$fw -w -C OUTPUT -m owner --uid-owner "\$uid" -j CI_SHOT 2>/dev/null ||
+    \$fw -w -I OUTPUT -m owner --uid-owner "\$uid" -j CI_SHOT
+done
+iptables -w -A CI_SHOT -d ${NET_ADDR} -j REJECT
+iptables -w -A CI_SHOT -d ${NET_CIDR} -j ACCEPT
+# systemd-resolved's stub; it forwards to the lab resolver itself.
+iptables -w -A CI_SHOT -d 127.0.0.53 -p udp --dport 53 -j ACCEPT
+iptables -w -A CI_SHOT -d 127.0.0.53 -p tcp --dport 53 -j ACCEPT
+for ns in \$(awk '/^nameserver/ && \$2 !~ /:/ {print \$2}' /etc/resolv.conf /run/systemd/resolve/resolv.conf 2>/dev/null); do
+  iptables -w -A CI_SHOT -d "\$ns" -p udp --dport 53 -j ACCEPT
+  iptables -w -A CI_SHOT -d "\$ns" -p tcp --dport 53 -j ACCEPT
+done
+for net in 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 224.0.0.0/3; do
+  iptables -w -A CI_SHOT -d "\$net" -j REJECT
+done
+for net in ::1/128 fc00::/7 fe80::/10 ff00::/8; do
+  ip6tables -w -A CI_SHOT -d "\$net" -j REJECT
+done
+EOF
+chmod +x /usr/local/sbin/ci-shot-firewall
+cat >/etc/systemd/system/ci-shot-firewall.service <<'EOF'
+[Unit]
+Description=Keep the screenshot browser away from this host and the lab network
+After=network-online.target incus.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/ci-shot-firewall
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+[[ -s /etc/timezone ]] || {
+  timedatectl set-timezone Europe/Vienna
+  echo Europe/Vienna >/etc/timezone
+}
+
+# The rewritten PR trees, served to the engine and to the containers. Waits
+# for incusbr0, which only exists once Incus is up.
+mkdir -p "$SERVE_ROOT"
+cat >/etc/systemd/system/ci-serve.service <<EOF
+[Unit]
+Description=Serve pull request trees to script builds
+After=incus.service network-online.target
+Wants=incus.service
+
+[Service]
+ExecStartPre=/bin/sh -c 'for i in \$(seq 1 60); do ip -4 addr show dev incusbr0 | grep -q ${NET_ADDR} && exit 0; incus network list >/dev/null 2>&1; sleep 1; done; exit 1'
+ExecStart=/usr/bin/python3 -m http.server ${SERVE_PORT} --bind ${NET_ADDR} --directory ${SERVE_ROOT}
+DynamicUser=yes
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Leftovers of a job that was killed before its cleanup trap ran.
+cat >/usr/local/sbin/ci-sweep <<'EOF'
+#!/usr/bin/env bash
+# Delete CI containers (ci-9000 .. ci-9099) that no running job holds, and stale trees.
+# A job holds its guest with /run/ci-jobs/<id>. A job killed outright (runner
+# restart, timeout) never removes it, so a lock older than any job can run
+# (90 minutes) holds nothing.
+held() { [[ -e "/run/ci-jobs/$1" && -z "$(find "/run/ci-jobs/$1" -mmin +180)" ]]; }
+for name in $(incus list -c n -f csv 2>/dev/null | grep -E '^ci-9[0-9]{3}$'); do
+  held "$name" && continue
+  incus delete "$name" --force &>/dev/null && echo "swept $name"
+done
+find /run/ci-jobs -maxdepth 1 -type f ! -name ".*" -mmin +180 -delete 2>/dev/null
+find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} +
+find /usr/local/community-scripts/logs /var/log/community-scripts -type f -mtime +2 -delete 2>/dev/null
+find /opt/actions-runner/*/_diag -type f -mtime +14 -delete 2>/dev/null
+true
+EOF
+chmod +x /usr/local/sbin/ci-sweep
+cat >/etc/systemd/system/ci-sweep.service <<'EOF'
+[Unit]
+Description=Remove leftover CI containers
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ci-sweep
+EOF
+cat >/etc/systemd/system/ci-sweep.timer <<'EOF'
+[Unit]
+Description=Remove leftover CI containers every 3 hours
+
+[Timer]
+OnCalendar=0/3:30
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+
+systemctl daemon-reload
+systemctl enable --now ci-serve.service ci-sweep.timer ci-shot-firewall.service >/dev/null
+systemctl restart ci-serve.service ci-shot-firewall.service
+
+# --- Runner (one per repository) ----------------------------------------------
+if [[ $# -eq 2 ]]; then
+  repo="$1"
+  token="$2"
+  [[ "$repo" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad repo" >&2; exit 2; }
+  dir="/opt/actions-runner/${repo#*/}"
+  mkdir -p "$dir"
+  cd "$dir"
+  if [[ ! -x ./config.sh ]]; then
+    curl -fsSL "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -xz
+    ./bin/installdependencies.sh >/dev/null
+  fi
+  if [[ -f .runner ]]; then
+    ./svc.sh stop >/dev/null 2>&1 || true
+    ./svc.sh uninstall >/dev/null 2>&1 || true
+    RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token "$token" >/dev/null 2>&1 || true
+  fi
+  # Only `incus-test`: see pve-ci-host-setup.sh for why there are no default labels.
+  RUNNER_ALLOW_RUNASROOT=1 ./config.sh --unattended --replace --no-default-labels \
+    --url "https://github.com/${repo}" --token "$token" \
+    --name "$(hostname -s)-${repo#*/}" --labels incus-test --work _work
+  ./svc.sh install root >/dev/null
+  ./svc.sh start >/dev/null
+  echo "runner for ${repo} registered"
+fi
+
+echo "incus test host ready: ${SERVE_ROOT} on ${NET_ADDR}:${SERVE_PORT}, sweeper every 3h"

.github/workflows/scripts/pve-ci-host-setup.sh

@@ -102,8 +102,9 @@ systemctl restart dnsmasq
   echo Europe/Vienna >/etc/timezone
 }
 
-# Containers live on the root filesystem; there is no LVM in a cloud-image PVE.
-pvesm set local --content iso,vztmpl,backup,rootdir,images
+# Containers and VM disks live on the root filesystem; there is no LVM in a
+# cloud-image PVE. Snippets hold the Cloud-Init configs some vm/ scripts write.
+pvesm set local --content iso,vztmpl,backup,rootdir,images,snippets
 
 # Templates are fetched once here rather than on every first build.
 pveam update >/dev/null
@@ -130,12 +131,26 @@ EOF
 # Leftovers of a job that was killed before its cleanup trap ran.
 cat >/usr/local/sbin/ci-sweep <<'EOF'
 #!/usr/bin/env bash
-# Destroy CI containers (9000-9099) that no running job holds, and stale trees.
+# Destroy CI containers (9000-9099) and VMs (9100-9199) that no running job
+# holds, and stale trees.
+# A job holds its guest with /run/ci-jobs/<id>. A job killed outright (runner
+# restart, timeout) never removes it, so a lock older than any job can run
+# (90 minutes) holds nothing.
+held() { [[ -e "/run/ci-jobs/$1" && -z "$(find "/run/ci-jobs/$1" -mmin +180)" ]]; }
 for id in $(pct list | awk 'NR>1 && $1>=9000 && $1<=9099 {print $1}'); do
-  [[ -e "/run/ci-jobs/$id" ]] && continue
+  held "$id" && continue
   pct stop "$id" --skiplock 1 &>/dev/null || true
   pct destroy "$id" --purge 1 --force 1 &>/dev/null && echo "swept CT $id"
 done
+for id in $(qm list | awk 'NR>1 && $1>=9100 && $1<=9199 {print $1}'); do
+  held "$id" && continue
+  qm stop "$id" --skiplock 1 --timeout 30 &>/dev/null || true
+  qm destroy "$id" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null && echo "swept VM $id"
+done
+# VM images and ISOs are cached between builds; drop the ones no build used
+# for a week. Container templates (*.tar.*) stay.
+find /var/lib/vz/template/cache /var/lib/vz/template/iso -maxdepth 1 -type f ! -name '*.tar.*' -atime +7 -delete 2>/dev/null
+find /run/ci-jobs -maxdepth 1 -type f ! -name ".*" -mmin +180 -delete 2>/dev/null
 find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} +
 # Engine logs that a killed job never got to remove.
 find /tmp -maxdepth 1 -type f \( -name 'create-lxc-*.log' -o -name 'pct_create_*.log' \

.github/workflows/scripts/pve-run-script.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
-# Build one ct/ script from a pull request on this Proxmox host, check that the
-# container really works, destroy it, and leave a result.json behind.
+# Build one ct/ or vm/ script from a pull request on this test host, check
+# that the container or VM really works, destroy it, and leave a result.json
+# behind. The vm/ part lives in pve-run-vm.sh, sourced from here.
+#
+# PLATFORM says what this host is: pve (Proxmox VE, pct) or incus. The ct/
+# scripts are the same for both; community-scripts/core picks its Incus
+# backend by itself when it finds incus instead of pveversion. Only the way
+# this harness reaches into the container differs (guest_* below).
 #
 # Runs on the self-hosted runner, which *is* the disposable test host. The
 # harness comes from the default branch; only the script under test comes from
@@ -17,10 +23,17 @@ set -uo pipefail
 
 : "${PR_REPO:?}" "${PR_SHA:?}" "${SCRIPT:?}" "${UPSTREAM:?}" "${CTID_BASE:?}" "${OUT_DIR:?}"
 OS="${OS:-}"
+PLATFORM="${PLATFORM:-pve}"
 CTID_COUNT="${CTID_COUNT:-50}"
 SERVE_ROOT="${SERVE_ROOT:-/srv/ci}"
-SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}"
-BRIDGE="${BRIDGE:-vmbr1}"
+# The containers' own network: vmbr1 + dnsmasq on Proxmox, incusbr0 on Incus.
+if [[ "$PLATFORM" == "incus" ]]; then
+  SERVE_URL="${SERVE_URL:-http://10.98.0.1:18080}"
+  BRIDGE="${BRIDGE:-incusbr0}"
+else
+  SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}"
+  BRIDGE="${BRIDGE:-vmbr1}"
+fi
 BUILD_TIMEOUT="${BUILD_TIMEOUT:-45m}"
 URL_WAIT="${URL_WAIT:-120}"
 
@@ -28,8 +41,10 @@ URL_WAIT="${URL_WAIT:-120}"
 [[ "$PR_REPO" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad PR_REPO" >&2; exit 2; }
 [[ "$UPSTREAM" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad UPSTREAM" >&2; exit 2; }
 [[ "$PR_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "bad PR_SHA" >&2; exit 2; }
-[[ "$SCRIPT" =~ ^ct/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; }
+[[ "$SCRIPT" =~ ^(ct|vm)/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; }
 [[ -z "$OS" || "$OS" =~ ^(debian|ubuntu|alpine)$ ]] || { echo "bad OS" >&2; exit 2; }
+[[ "$PLATFORM" =~ ^(pve|incus)$ ]] || { echo "bad PLATFORM" >&2; exit 2; }
+[[ "$PLATFORM" == "pve" || "$SCRIPT" == ct/* ]] || { echo "vm/ scripts run on pve only" >&2; exit 2; }
 [[ "$CTID_BASE" =~ ^[0-9]+$ ]] || { echo "bad CTID_BASE" >&2; exit 2; }
 
 mkdir -p "$OUT_DIR"
@@ -41,8 +56,9 @@ UPDATE_RAW="$OUT_DIR/update.raw.log"
 UPDATE_LOG="$OUT_DIR/update.log"
 UPDATE_TIMEOUT="${UPDATE_TIMEOUT:-30m}"
 
+kind="${SCRIPT%%/*}" # ct or vm
 app=$(basename "$SCRIPT" .sh)
-job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${app}-${OS:-default}"
+job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${PLATFORM}-${kind}-${app}-${OS:-default}"
 tree="$SERVE_ROOT/$job"
 base="$SERVE_URL/$job"
 started=$(date +%s)
@@ -50,7 +66,30 @@ started=$(date +%s)
 status="fail"
 step=""
 exit_code=""
-ctid=""
+ctid="" # the guest's ID (pve: 9000, incus: ci-9000), container or VM
+
+# The container, whatever the platform. ctid is set once a slot is reserved.
+guest_exec() {
+  if [[ "$PLATFORM" == "incus" ]]; then incus exec "$ctid" -- "$@"; else pct exec "$ctid" -- "$@"; fi
+}
+guest_exists() {
+  if [[ "$PLATFORM" == "incus" ]]; then incus info "$ctid" &>/dev/null; else pct status "$ctid" &>/dev/null; fi
+}
+guest_running() {
+  if [[ "$PLATFORM" == "incus" ]]; then
+    [[ "$(incus list "^${ctid}\$" -c s -f csv 2>/dev/null)" == "RUNNING" ]]
+  else
+    pct status "$ctid" 2>/dev/null | grep -q running
+  fi
+}
+guest_destroy() {
+  if [[ "$PLATFORM" == "incus" ]]; then
+    incus delete "$ctid" --force &>/dev/null || true
+  else
+    pct stop "$ctid" --skiplock 1 &>/dev/null || true
+    pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true
+  fi
+}
 ct_ip=""
 url=""
 http_code=""
@@ -60,8 +99,13 @@ failed_units=""
 update_status=""
 update_step=""
 
+# Spinner frames: the engine redraws "⠋ Starting VM" a few times a second
+# without a tty too. An alternation, not a bracket: these are multibyte and
+# the runner's locale may be C.
+SPINNER_RE='^\s*(⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏)'
+
 strip_ansi() {
-  sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$'
+  sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' | grep -vE "$SPINNER_RE"
 }
 
 # The install script's own output: everything $STD hides from the build log.
@@ -71,11 +115,12 @@ strip_ansi() {
 collect_install_log() {
   [[ -n "$ctid" ]] || return 0
   # Exactly one build ran in this container, so the newest install log is ours.
-  if pct status "$ctid" &>/dev/null &&
-    pct exec "$ctid" -- sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null |
+  if guest_exists &&
+    guest_exec sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null |
     strip_ansi >"$INSTALL_LOG_OUT" && [[ -s "$INSTALL_LOG_OUT" ]]; then
     return 0
   fi
+  [[ "$PLATFORM" == "pve" ]] || { rm -f "$INSTALL_LOG_OUT"; return 0; }
   # The engine names its host-side files after the session: pct_create_<ctid>_<date>_<session>.log
   local sid f
   sid=$(find /tmp -maxdepth 1 -name "pct_create_${ctid}_*.log" -newer "$RAW_LOG.start" -printf '%f\n' 2>/dev/null |
@@ -120,15 +165,15 @@ finish() {
   [[ -f "$UPDATE_RAW" ]] && strip_ansi <"$UPDATE_RAW" >"$UPDATE_LOG"
   rm -f "$UPDATE_RAW"
   jq -n \
-    --arg script "$SCRIPT" --arg os "${OS:-default}" --arg status "$status" \
+    --arg script "$SCRIPT" --arg kind "$kind" --arg platform "$PLATFORM" --arg os "${OS:-default}" --arg status "$status" \
     --arg step "$step" --arg exit_code "$exit_code" --arg sha "$PR_SHA" \
     --arg ctid "$ctid" --arg url "$url" --arg http_code "$http_code" \
-    --arg failed_units "$failed_units" \
+    --arg failed_units "$failed_units" --arg ip "$ct_ip" \
     --arg update_status "$update_status" --arg update_step "$update_step" \
     --argjson duration "$(($(date +%s) - started))" \
     --arg tail "$([[ -f "$LOG" ]] && tail -n 60 "$LOG")" \
-    '{script: $script, os: $os, status: $status, step: $step, exit_code: $exit_code,
-      sha: $sha, ctid: $ctid, url: $url, http_code: $http_code,
+    '{script: $script, kind: $kind, platform: $platform, os: $os, status: $status, step: $step, exit_code: $exit_code,
+      sha: $sha, ctid: $ctid, ip: $ip, url: $url, http_code: $http_code,
       failed_units: $failed_units, update_status: $update_status, update_step: $update_step,
       duration: $duration, log_tail: $tail}' >"$RESULT"
   echo "::group::result"
@@ -150,7 +195,7 @@ install_pid=""
 
 # Colours stay, cursor movement and spinner redraws go.
 live_filter() {
-  sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E '/^\s*$/d'
+  sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E "/^\s*\$/d; /${SPINNER_RE}/d"
 }
 
 start_streams() {
@@ -158,17 +203,21 @@ start_streams() {
   echo "::stop-commands::${stream_token}"
   tail -n +1 -F --pid="$build_pid" "$RAW_LOG" 2>/dev/null > >(live_filter) &
   log_pid=$!
+  # A VM has no install log to follow: its script runs on this host only.
+  [[ "$kind" == "ct" ]] || return 0
   # setsid: its own process group, so stop_streams can take down the
-  # pct exec / tail pair inside it in one go.
+  # pct/incus exec + tail pair inside it in one go.
   setsid bash -c '
+    gx() { if [ "$2" = incus ]; then incus exec "$1" -- "${@:3}"; else pct exec "$1" -- "${@:3}"; fi; }
     for _ in $(seq 1 900); do
-      f=$(pct exec "$1" -- sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null)
+      f=$(gx "$1" "$2" sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null)
       if [ -n "$f" ]; then
         echo "──────── install log of CT $1 (live) ────────"
+        if [ "$2" = incus ]; then exec incus exec "$1" -- tail -n +1 -F "$f"; fi
         exec pct exec "$1" -- tail -n +1 -F "$f"
       fi
       sleep 2
-    done' _ "$ctid" 2>/dev/null > >(live_filter | sed -u 's/^/  │ /') &
+    done' _ "$ctid" "$PLATFORM" 2>/dev/null > >(live_filter | sed -u 's/^/  │ /') &
   install_pid=$!
 }
 
@@ -195,15 +244,14 @@ cleanup() {
     kill -TERM "$pid" &>/dev/null || true
     sleep 2
   done
-  if [[ -n "$ctid" ]]; then
-    if pct status "$ctid" &>/dev/null; then
-      pct stop "$ctid" --skiplock 1 &>/dev/null || true
-      pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true
-    fi
+  if [[ -n "$ctid" && "$kind" == "vm" ]]; then
+    vm_cleanup
+  elif [[ -n "$ctid" ]]; then
+    guest_exists && guest_destroy
     rm -f "/run/ci-jobs/$ctid"
     # What the engine leaves on the host for this container and its session.
     local f sid
-    for f in /tmp/pct_create_"${ctid}"_*.log; do
+    [[ "$PLATFORM" == "pve" ]] && for f in /tmp/pct_create_"${ctid}"_*.log; do
       [[ -f "$f" && "$f" -nt "$RAW_LOG.start" ]] || continue
       sid=$(sed -E 's/.*_([0-9a-f]+)\.log$/\1/' <<<"$f")
       rm -f -- "$f" /tmp/create-lxc-"${sid}".log /tmp/*-"${ctid}"-"${sid}".log
@@ -245,19 +293,33 @@ setup_error() {
 # --- Reserve a container ID ---------------------------------------------------
 # A lock file per ID, not just "first free one", because the engine only creates
 # the container minutes later and two jobs would otherwise pick the same ID.
+# VMs take the range 100 above the containers' (DevScripts 9100-9149,
+# ProxmoxVE 9150-9199).
 mkdir -p /run/ci-jobs
+if [[ "$kind" == "vm" ]]; then
+  source "$(dirname "$0")/pve-run-vm.sh"
+  CTID_BASE=$((CTID_BASE + 100))
+  vm_wait_turn
+fi
 exec 9>/run/ci-jobs/.lock
 flock 9
+# On Incus the ID is an instance name: var_ctid=ci-9000 names the container.
 for ((id = CTID_BASE; id < CTID_BASE + CTID_COUNT; id++)); do
-  [[ -e "/run/ci-jobs/$id" ]] && continue
-  pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue
-  echo "$job" >"/run/ci-jobs/$id"
-  ctid=$id
+  name=$id
+  [[ "$PLATFORM" == "incus" ]] && name="ci-$id"
+  [[ -e "/run/ci-jobs/$name" ]] && continue
+  if [[ "$PLATFORM" == "incus" ]]; then
+    incus info "$name" &>/dev/null && continue
+  else
+    pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue
+  fi
+  echo "$job" >"/run/ci-jobs/$name"
+  ctid=$name
   break
 done
 flock -u 9
 exec 9>&-
-[[ -n "$ctid" ]] || setup_error "no free container ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))"
+[[ -n "$ctid" ]] || setup_error "no free ${kind} ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))"
 
 # --- Fetch the approved commit and point it at ourselves ------------------------
 mkdir -p "$tree"
@@ -269,17 +331,26 @@ upstream_re="https://raw\.githubusercontent\.com/${UPSTREAM//./\\.}/(refs/heads/
 grep -rlE "$upstream_re" "$tree" --include='*.sh' --include='*.func' |
   xargs -r sed -i -E "s#${upstream_re}#${base}#g"
 
+if [[ "$kind" == "vm" ]]; then
+  vm_build_and_check
+  exit 0
+fi
+
 # --- Build ----------------------------------------------------------------------
-echo "Building ${SCRIPT} (${OS:-default OS}) as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}"
+echo "Building ${SCRIPT} (${OS:-default OS}) on ${PLATFORM} as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}"
 hostname_safe="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-50)"
 build_env=(
   PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
   HOME=/root TERM=dumb LANG=C.UTF-8
   mode=default MODE=default PHS_SILENT=1 DIAGNOSTICS=no
-  var_ctid="$ctid" var_hostname="$hostname_safe"
-  var_container_storage=local var_template_storage=local var_brg="$BRIDGE"
+  var_ctid="$ctid" var_hostname="$hostname_safe" var_brg="$BRIDGE"
   "dev_mode=timing,keep" COMMUNITY_SCRIPTS_URL="$base"
 )
+if [[ "$PLATFORM" == "incus" ]]; then
+  build_env+=(var_container_storage=default)
+else
+  build_env+=(var_container_storage=local var_template_storage=local)
+fi
 touch "$RAW_LOG.start"
 [[ -n "$OS" ]] && build_env+=(var_os="$OS")
 # Both engines turn var_github_token into GITHUB_TOKEN inside the container.
@@ -414,9 +485,9 @@ fi
 grep -qi 'completed successfully' "$RAW_LOG" || fail "build exited 0 but never reported success"
 
 # --- Checks ---------------------------------------------------------------------
-pct status "$ctid" 2>/dev/null | grep -q running || fail "container ${ctid} is not running"
+guest_running || fail "container ${ctid} is not running"
 
-ct_ip=$(pct exec "$ctid" -- sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1)
+ct_ip=$(guest_exec sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1)
 
 # What the app's page looks like, for the PR comment. Chromium runs as its own
 # unprivileged user with its sandbox on: the page is served by code under test.
@@ -450,11 +521,11 @@ health_problem=""
 check_health() {
   health_problem=""
   failed_units=""
-  if pct exec "$ctid" -- sh -c 'command -v systemctl' &>/dev/null; then
+  if guest_exec sh -c 'command -v systemctl' &>/dev/null; then
     sleep 10
-    failed_units=$(pct exec "$ctid" -- systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -)
-  elif pct exec "$ctid" -- sh -c 'command -v rc-status' &>/dev/null; then
-    failed_units=$(pct exec "$ctid" -- rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -)
+    failed_units=$(guest_exec systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -)
+  elif guest_exec sh -c 'command -v rc-status' &>/dev/null; then
+    failed_units=$(guest_exec rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -)
   fi
   [[ -z "$failed_units" ]] || { health_problem="failed services: ${failed_units}"; return 1; }
   [[ -n "$url" ]] || return 0
@@ -486,14 +557,14 @@ update_fail() {
   fail "update: $1"
 }
 
-version_files=$(pct exec "$ctid" -- sh -c '
+version_files=$(guest_exec sh -c '
   for f in /root/.*; do
     [ -f "$f" ] || continue
     case "$f" in */.install-* | *.pending | *.bak | */.bash* | */.profile | */.hushlogin | */.viminfo | */.lesshst | */.wget-hsts | */.selected_editor) continue ;; esac
     [ "$(wc -l <"$f")" -le 1 ] && grep -qE "^v?[0-9]+(\.[0-9A-Za-z_+-]+)+$" "$f" && echo "$f"
   done' 2>/dev/null | grep -E '^/root/\.[A-Za-z0-9._-]+$')
 for f in $version_files; do
-  pct exec "$ctid" -- sh -c "echo '$UPDATE_FAKE' > '$f'; rm -f '$f.pending'" ||
+  guest_exec sh -c "echo '$UPDATE_FAKE' > '$f'; rm -f '$f.pending'" ||
     update_fail "could not reset $f"
 done
 [[ -n "$version_files" ]] && echo "Update test: set back $(tr '\n' ' ' <<<"$version_files")to ${UPDATE_FAKE}"
@@ -502,7 +573,9 @@ done
 update_token=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')
 echo "──────── update test (live) ────────"
 echo "::stop-commands::${update_token}"
-timeout --kill-after=1m "$UPDATE_TIMEOUT" pct exec "$ctid" -- env PHS_SILENT=1 mode=default MODE=default \
+# timeout runs a program, not a shell function, so spell out the exec here.
+if [[ "$PLATFORM" == "incus" ]]; then guest_cmd=(incus exec "$ctid" --); else guest_cmd=(pct exec "$ctid" --); fi
+timeout --kill-after=1m "$UPDATE_TIMEOUT" "${guest_cmd[@]}" env PHS_SILENT=1 mode=default MODE=default \
   TERM=dumb DIAGNOSTICS=no var_ignore_breaking_changes=yes ${GH_API_TOKEN:+GITHUB_TOKEN="$GH_API_TOKEN"} update </dev/null \
   > >(tee "$UPDATE_RAW" | live_filter) 2>&1 &
 update_pid=$!
@@ -538,7 +611,7 @@ fi
 if strip_ansi <"$UPDATE_RAW" | grep -qi 'updated successfully'; then
   update_status="updated"
   for f in $version_files; do
-    [[ "$(pct exec "$ctid" -- cat "$f" 2>/dev/null)" != "$UPDATE_FAKE" ]] ||
+    [[ "$(guest_exec cat "$f" 2>/dev/null)" != "$UPDATE_FAKE" ]] ||
       update_fail "reported success, but $f is still ${UPDATE_FAKE}"
   done
 else

.github/workflows/scripts/pve-run-vm.sh

@@ -0,0 +1,140 @@
+# shellcheck shell=bash
+# The vm/ half of pve-run-script.sh, sourced from it: same inputs, same
+# result.json, same live output. Only building and checking differ.
+#
+# A VM script runs entirely on this host (there is no install script inside
+# the guest to follow), unattended through the core engine's VM_UNATTENDED=1:
+# the script's defaults, with ID, name, bridge and storage set from here. The
+# guest then boots on vmbr1 like the containers do, so its address is whatever
+# this host's dnsmasq leased to its MAC; no guest agent needed.
+#
+# What passes: the script exits 0 and says "Completed successfully", the VM is
+# running and keeps running through the boot wait, and where the script prints
+# a URL built from the VM's address (http://${VM_IP}:9000) that URL answers 200.
+# An installer ISO or an image without DHCP gets no address; that is reported,
+# not failed. There is no update test: a VM has no `update`.
+
+VM_BOOT_WAIT="${VM_BOOT_WAIT:-300}"
+VM_URL_WAIT="${VM_URL_WAIT:-600}"
+LEASES="${LEASES:-/var/lib/misc/dnsmasq.leases}"
+
+# One VM build per host at a time: a VM takes 2-8 GB of RAM and a disk image,
+# where a container takes a fraction. Held until the job exits; the build is
+# started with this descriptor closed, so a QEMU process can never inherit it.
+vm_wait_turn() {
+  exec 7>/run/ci-jobs/.vm-turn
+  flock -n 7 && return 0
+  echo "Another VM build is running on this host; waiting for it (one VM at a time)"
+  flock -w 3600 7 || setup_error "waited an hour for this host's VM slot"
+}
+
+vm_cleanup() {
+  if qm status "$ctid" &>/dev/null; then
+    qm stop "$ctid" --skiplock 1 --timeout 30 &>/dev/null || true
+    qm destroy "$ctid" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null || true
+  fi
+  rm -f "/run/ci-jobs/$ctid" "/run/ci-jobs/console-${ctid}.png"
+  # Cloud-Init network snippets the script wrote for this VM.
+  [[ -f "$RAW_LOG.start" ]] &&
+    find /var/lib/vz/snippets -maxdepth 1 -type f -name "*${ctid}*" -newer "$RAW_LOG.start" -delete 2>/dev/null
+  true
+}
+
+# The VM's screen, straight from QEMU. Pixels only, but they come from the
+# guest, so the same checks as for a browser screenshot apply.
+vm_console_shot() {
+  local f="/run/ci-jobs/console-${ctid}.png"
+  rm -f "$f"
+  printf 'screendump %s -f png\n' "$f" | timeout 30 qm monitor "$ctid" &>/dev/null
+  if [[ -s "$f" && $(stat -c %s "$f") -lt 3000000 ]] &&
+    [[ "$(head -c 8 "$f" | od -An -tx1 | tr -d ' \n')" == "89504e470d0a1a0a" ]]; then
+    cp "$f" "$OUT_DIR/screenshot.png"
+  fi
+  rm -f "$f"
+}
+
+vm_running() {
+  qm status "$ctid" 2>/dev/null | grep -q 'status: running'
+}
+
+vm_build_and_check() {
+  local hn mac pattern waited last
+  hn="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-40)-${ctid}"
+  echo "Building ${SCRIPT} as VM ${ctid} from ${PR_REPO}@${PR_SHA:0:7}"
+  local vm_env=(
+    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+    HOME=/root TERM=dumb LANG=C.UTF-8 DIAGNOSTICS=no
+    VM_UNATTENDED=1 VM_VMID="$ctid" VM_HOSTNAME="$hn" VM_BRIDGE="$BRIDGE"
+    VM_STORAGE=local VM_ISO_STORAGE=local VM_START=yes
+    COMMUNITY_SCRIPTS_URL="$base"
+  )
+  # Release lookups (vm_latest_from_github and the like).
+  [[ -n "${GH_API_TOKEN:-}" ]] && vm_env+=(GITHUB_TOKEN="$GH_API_TOKEN")
+  touch "$RAW_LOG.start"
+
+  env -i "${vm_env[@]}" timeout --kill-after=1m "$BUILD_TIMEOUT" \
+    bash -c 'bash -c "$(curl -fsSL "$1")"' _ "$base/$SCRIPT" </dev/null >"$RAW_LOG" 2>&1 7>&- &
+  build_pid=$!
+  start_streams
+  wait "$build_pid"
+  exit_code=$?
+  build_pid=""
+  stop_streams
+
+  ((exit_code == 124 || exit_code == 137)) && fail "build timed out after ${BUILD_TIMEOUT}"
+  if ((exit_code != 0)); then
+    last=$(failure_line)
+    step="script exited ${exit_code}${last:+: $last}"
+    status="fail"
+    classify_cpu_failure
+    fail "$step" "$status"
+  fi
+  # vm_finish ends on a msg_ok of the script's choosing ("VM created. ..."),
+  # older scripts on "Completed successfully". exit_script and friends leave
+  # with 0 too, but on a cross.
+  last=$(strip_ansi <"$RAW_LOG" | tail -n 1 | cut -c1-200)
+  if ! grep -qi 'completed successfully' "$RAW_LOG" && ! [[ "$last" =~ ^[[:space:]]*✔ ]]; then
+    fail "script exited 0 but never reported success${last:+ (last line: ${last})}"
+  fi
+  vm_running || fail "VM ${ctid} is not running after the script finished"
+
+  # The address dnsmasq handed to this VM's MAC. The VM must stay up while it
+  # boots: one that powers itself off or crashes is a failure.
+  mac=$(qm config "$ctid" 2>/dev/null | sed -nE 's/^net0:[^=]*=([0-9A-Fa-f:]{17}).*/\1/p' | tr 'A-F' 'a-f')
+  echo "Waiting up to ${VM_BOOT_WAIT}s for VM ${ctid} (${mac:-no NIC}) to boot and get an address"
+  for ((waited = 0; waited < VM_BOOT_WAIT; waited += 5)); do
+    vm_running || fail "VM ${ctid} stopped by itself while booting"
+    [[ -n "$mac" ]] && ct_ip=$(awk -v m="$mac" '$2 == m {print $3}' "$LEASES" 2>/dev/null | tail -n 1)
+    [[ -n "$ct_ip" ]] && break
+    sleep 5
+  done
+  [[ -n "$ct_ip" ]] && echo "VM ${ctid} got ${ct_ip}" || echo "VM ${ctid} got no DHCP lease within ${VM_BOOT_WAIT}s"
+
+  # Only a URL the script builds from the address it found; <VM-IP> in a
+  # summary is a hint for later (after an installer ran), not a promise.
+  # ${VM_IP}, $VM_IP and ${VM_IP:-<VM-IP>} all count.
+  local ip_re='\$\{?VM_IP(:-[^}]*)?\}?'
+  pattern=$(grep -oE "https?://${ip_re}(:[0-9]+)?(/[A-Za-z0-9._~/?=&%-]*)?" "$tree/$SCRIPT" | head -n1)
+  if [[ -n "$pattern" ]]; then
+    [[ -n "$ct_ip" ]] || fail "no IP address within ${VM_BOOT_WAIT}s, so ${pattern} could not be checked"
+    url=$(sed -E "s#${ip_re}#${ct_ip}#" <<<"$pattern")
+    for ((waited = 0; waited < VM_URL_WAIT; waited += 10)); do
+      vm_running || fail "VM ${ctid} stopped by itself while its web UI was awaited"
+      http_code=$(curl -sk -L --max-redirs 5 -o /dev/null -w '%{http_code}' --max-time 10 "$url" || true)
+      [[ "$http_code" =~ ^2[0-9][0-9]$ ]] && break
+      sleep 10
+    done
+    [[ "$http_code" =~ ^[1-5][0-9][0-9]$ ]] && take_screenshot screenshot.png
+  else
+    # Let the console settle on a login prompt rather than a boot message.
+    sleep 20
+  fi
+  [[ -s "$OUT_DIR/screenshot.png" ]] || vm_console_shot
+  if [[ -n "$url" && ! "$http_code" =~ ^2[0-9][0-9]$ ]]; then
+    fail "${url} answered ${http_code/#000/nothing} instead of 200 within ${VM_URL_WAIT}s"
+  fi
+  vm_running || fail "VM ${ctid} stopped by itself"
+
+  status="pass"
+  finish
+}