From 01f37f501fcbcae8a330ce024c16da1d7638333d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B6gl-Brunner=20Michel?= Date: Wed, 11 Nov 2026 11:48:57 +0100 Subject: [PATCH] pr-script-test: build vm/ scripts, and every ct/ script on Incus too - vm/ scripts run unattended (VM_UNATTENDED=1) on the Proxmox hosts, IDs 9100-9199, one VM per host at a time. Checks: success line, the VM keeps running, a DHCP lease, and a ${VM_IP} URL answering 200. Console screenshot from QEMU when there is no web UI. ARM64-only scripts are listed as not built. - Every ct/ script now builds on pve and incus (new incus-test runners, incus-ci-host-setup.sh); the comment gets a Platform column. - Spinner frames are dropped from logs and live output. - The sweeper ignores and removes locks of killed jobs (older than 3h). --- .github/workflows/pr-script-test.yml | 110 ++++++---- .../workflows/scripts/incus-ci-host-setup.sh | 193 ++++++++++++++++++ .../workflows/scripts/pve-ci-host-setup.sh | 23 ++- .github/workflows/scripts/pve-run-script.sh | 157 ++++++++++---- .github/workflows/scripts/pve-run-vm.sh | 140 +++++++++++++ 5 files changed, 538 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/scripts/incus-ci-host-setup.sh create mode 100644 .github/workflows/scripts/pve-run-vm.sh diff --git a/.github/workflows/pr-script-test.yml b/.github/workflows/pr-script-test.yml index 10590d60c..ae8f51b5d 100644 --- a/.github/workflows/pr-script-test.yml +++ b/.github/workflows/pr-script-test.yml @@ -1,6 +1,6 @@ name: PR script test -# Builds the ct/ scripts a pull request touches on a real Proxmox host and +# Builds the ct/ and vm/ scripts a pull request touches on real Proxmox and Incus hosts and # reports the outcome on the PR, as a check run and as one comment kept up to # date. # @@ -16,7 +16,8 @@ name: PR script test # even one that lands in the seconds before this job runs. # # gate ubuntu-latest who asked, which scripts, which commit -# test proxmox-test build + checks, one matrix job per script/OS +# test proxmox-test / build + checks, one matrix job per script, platform +# incus-test (pve or incus) and OS # report ubuntu-latest check run + PR comment # # The test job gets a read-only token and never sees a secret: the host it runs @@ -46,7 +47,7 @@ on: required: false type: string scripts: - description: "Space-separated ct/ paths (empty = everything the PR touches)" + description: "Space-separated ct/ or vm/ paths (empty = everything the PR touches)" required: false type: string @@ -171,8 +172,9 @@ jobs: core.notice(why); }; if (isIssue) { - // App names (bark-server) or ct/ paths; the default branch is trusted. - entries = tokens.map(t => /^ct\/[A-Za-z0-9._-]+\.sh$/.test(t) ? t + // App names (bark-server) or ct/ and vm/ paths; the default branch + // is trusted. A name without a ct/ script falls back to vm/ below. + entries = tokens.map(t => /^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(t) ? t : /^[A-Za-z0-9._-]+$/.test(t) ? `ct/${t.replace(/\.sh$/, '')}.sh` : t); if (entries.length === 0) { await refuse('In an issue, `/test` needs to know which app.', 'Comment `/test `, for example `/test bark-server` (the name of its `ct/` script).'); @@ -202,10 +204,10 @@ jobs: return; } } - if (entries.some(e => !/^ct\/[A-Za-z0-9._-]+\.sh$/.test(e))) { + if (entries.some(e => !/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(e))) { await refuse( - isIssue ? 'Only app names (as in `ct/.sh`) can be tested.' : 'Only `ct/.sh` paths can be requested.', - isIssue ? 'For example: `/test bark-server`' : 'For example: `/test ct/bark-server.sh`, or just `/test`.'); + isIssue ? 'Only app names (as in `ct/.sh` or `vm/.sh`) can be tested.' : 'Only `ct/.sh` and `vm/.sh` paths can be requested.', + isIssue ? 'For example: `/test bark-server` or `/test debian-vm`' : 'For example: `/test ct/bark-server.sh`, or just `/test`.'); return; } // Everything the PR changes that this test cannot build is named in @@ -219,29 +221,48 @@ jobs: if (f.status === 'removed') continue; let m; if ((m = f.filename.match(/^install\/([A-Za-z0-9._-]+)-install\.sh$/))) set.add(`ct/${m[1]}.sh`); - else if (/^ct\/[A-Za-z0-9._-]+\.sh$/.test(f.filename)) set.add(f.filename); - else if (/^vm\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: \`vm/\` scripts are not covered`); - else if (/^(tools|turnkey)\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: only \`ct/\` scripts are built`); + else if (/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(f.filename)) set.add(f.filename); + else if (/^(tools|turnkey)\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: only \`ct/\` and \`vm/\` scripts are built`); } entries = [...set].sort(); } const include = []; - for (const script of entries.slice(0, MAX)) { - let text; + const fetchScript = async p => { try { - const res = await github.rest.repos.getContent({ owner: head.owner.login, repo: head.name, path: script, ref: headSha }); - text = Buffer.from(res.data.content || '', 'base64').toString('utf8'); - } catch (e) { + const res = await github.rest.repos.getContent({ owner: head.owner.login, repo: head.name, path: p, ref: headSha }); + return Buffer.from(res.data.content || '', 'base64').toString('utf8'); + } catch (e) { return null; } + }; + for (let script of entries.slice(0, MAX)) { + let text = await fetchScript(script); + // /test debian-vm in an issue: there is no ct/debian-vm.sh, but a vm/ one. + if (text === null && isIssue && script.startsWith('ct/')) { + const vm = `vm/${script.slice(3)}`; + if ((text = await fetchScript(vm)) !== null) script = vm; + } + if (text === null) { notBuilt.push(isIssue - ? `\`${script}\` does not exist on the default branch` + ? (script.startsWith('ct/') ? `${code(script.slice(3, -3))}: neither \`ct/\` nor \`vm/\` has it on the default branch` : `${code(script)} does not exist on the default branch`) : `\`${script}\` does not exist in this PR (an \`install/\` script is built through the \`ct/\` script of the same name)`); continue; } + const isVm = script.startsWith('vm/'); + // The test hosts are amd64; an ARM-only VM script refuses to run there. + if (isVm && /^\s*vm_require_arch\s+arm64\b/m.test(text)) { + notBuilt.push(`${code(script)}: ARM64 only, the test hosts are amd64`); + continue; + } // Scripts that ask for the OS get built once per OS. - const oses = /msg_menu\s+"Choose the container OS"/.test(text) ? ['debian', 'alpine'] : ['']; - const name = script.slice(3, -3); - for (const os of oses) include.push({ script, os, name: os ? `${name}-${os}` : name }); + const oses = !isVm && /msg_menu\s+"Choose the container OS"/.test(text) ? ['debian', 'alpine'] : ['']; + const name = isVm ? `vm-${script.slice(3, -3)}` : script.slice(3, -3); + // Every ct/ script on every platform: the same files run on Proxmox + // and, through community-scripts/core's Incus backend, on Incus. + // VMs are Proxmox only. + const platforms = isVm ? ['pve'] : ['pve', 'incus']; + for (const platform of platforms) { + for (const os of oses) include.push({ script, os, platform, name: `${os ? `${name}-${os}` : name}-${platform}` }); + } } for (const script of entries.slice(MAX)) notBuilt.push(`\`${script}\`: at most ${MAX} scripts per run, name it with \`/test ${script}\``); if (include.length === 0) { @@ -249,10 +270,10 @@ jobs: MARK, '### Proxmox script test', '', isIssue ? 'ℹ️ Nothing to build:' - : 'ℹ️ Nothing to build: this test builds `ct/` scripts (and changed `install/` scripts through their `ct/` script).', '', - ...(notBuilt.length ? notBuilt.map(n => `- ${n}`) : ['- this PR changes no `ct/` or `install/` script']), + : 'ℹ️ Nothing to build: this test builds `ct/` and `vm/` scripts (and changed `install/` scripts through their `ct/` script).', '', + ...(notBuilt.length ? notBuilt.map(n => `- ${n}`) : ['- this PR changes no `ct/`, `vm/` or `install/` script']), ].join('\n')); - core.notice('No ct/ script to build'); + core.notice('No ct/ or vm/ script to build'); return; } @@ -263,7 +284,7 @@ jobs: const { data: check } = await github.rest.checks.create({ owner, repo, name: 'Proxmox script test', head_sha: headSha, status: 'in_progress', details_url: runUrl, - output: { title: `Building ${include.length} on Proxmox`, summary: include.map(i => `- \`${i.script}\` ${i.os}`).join('\n') }, + output: { title: `Building ${include.length}`, summary: include.map(i => `- \`${i.script}\` ${[i.platform, i.os].filter(Boolean).join(' · ')}`).join('\n') }, }); checkId = String(check.id); } @@ -280,7 +301,7 @@ jobs: await sticky([ MARK, '### Proxmox script test', '', `⏳ Building ${include.length} at ${headSha.slice(0, 7)}${isIssue ? ' (default branch)' : ''} — [watch it live](${runUrl}) (open the \`test\` job)`, '', - ...include.map(i => `- \`${i.script}\`${i.os ? ` (${i.os})` : ''}`), + ...include.map(i => `- \`${i.script}\` (${[i.platform, i.os].filter(Boolean).join(' · ')})`), ...(notBuilt.length ? ['', '**Not built:**', ...notBuilt.map(n => `- ${n}`)] : []), ].join('\n')); @@ -289,7 +310,7 @@ jobs: if: needs.gate.outputs.matrix != '[]' && needs.gate.outputs.matrix != '' # Only `proxmox-test`, never `self-hosted`: the runner is registered without # default labels so that no other workflow can be scheduled onto it. - runs-on: proxmox-test + runs-on: ${{ matrix.platform == 'incus' && 'incus-test' || 'proxmox-test' }} permissions: contents: read timeout-minutes: 90 # install (45) + update (30) + checks @@ -311,6 +332,7 @@ jobs: PR_SHA: ${{ needs.gate.outputs.sha }} SCRIPT: ${{ matrix.script }} OS: ${{ matrix.os }} + PLATFORM: ${{ matrix.platform }} UPSTREAM: ${{ github.repository }} CTID_BASE: ${{ github.repository == 'community-scripts/ProxmoxVE' && '9050' || '9000' }} # Only so release lookups are not rate limited: unauthenticated, all @@ -390,7 +412,10 @@ jobs: r.update_log = fs.existsSync(ulog) ? fs.readFileSync(ulog, 'utf8') : ''; const ushot = path.join(dir, 'screenshot-update.png'); r.shot_update_file = fs.existsSync(ushot) ? ushot : ''; - found.set(`${r.script}|${r.os}`, r); + // Only the two platforms this workflow knows; an unknown one would + // never match a matrix row anyway. + if (!['pve', 'incus'].includes(r.platform)) r.platform = 'pve'; + found.set(`${r.script}|${r.platform}|${r.os}`, r); } catch (err) { core.warning(`unreadable ${p}: ${err.message}`); } } } @@ -398,12 +423,15 @@ jobs: walk('results'); // A job that died before writing result.json still gets a row. - const rows = matrix.map(m => ({ ...(found.get(`${m.script}|${m.os || 'default'}`) || { - script: m.script, os: m.os || 'default', + const rows = matrix.map(m => ({ ...(found.get(`${m.script}|${m.platform || 'pve'}|${m.os || 'default'}`) || { + script: m.script, os: m.os || 'default', platform: m.platform || 'pve', status: process.env.TEST_RESULT === 'cancelled' ? 'cancelled' : 'error', step: 'no result (runner or job failure)', duration: 0, log_tail: '', install_log: '', shot_file: '', update_status: '', update_log: '', shot_update_file: '', }), name: m.name })); + // Where a row ran: "pve", "incus · alpine", or "VM" (Proxmox only). + const isVm = r => r.script.startsWith('vm/'); + const where = r => isVm(r) ? 'VM' : r.os && r.os !== 'default' ? `${r.platform} · ${r.os}` : r.platform; // Screenshots go to an orphan branch, because a comment can only show // an image that has a URL. One file per PR and script, overwritten on @@ -434,7 +462,7 @@ jobs: catch (e) { if (e.status !== 404) throw e; } const { data } = await github.rest.repos.createOrUpdateFileContents({ owner, repo, branch: BRANCH, path: file, sha, content: buf.toString('base64'), - message: `Screenshot: ${r.script} (${r.os})${suffix} for #${prNum} at ${process.env.SHA.slice(0, 7)}`, + message: `Screenshot: ${r.script} (${where(r)})${suffix} for #${prNum} at ${process.env.SHA.slice(0, 7)}`, }); return `https://raw.githubusercontent.com/${owner}/${repo}/${data.commit.sha}/${file}`; } catch (e) { @@ -482,13 +510,15 @@ jobs: const updateIcon = { updated: '✅', 'no-update': '➖', fail: '❌', error: '⚠️' }; const detail = r => { if (r.status !== 'pass') return clean(r.step); + const ip = /^[0-9.]{7,15}$/.test(r.ip || '') ? r.ip : ''; + if (isVm(r) && !r.url) return ip ? `booted, got ${ip}` : 'booted, no DHCP lease (an installer ISO, or no network)'; const reach = r.url ? `${clean(r.url)} → ${r.http_code}` : 'no URL to check'; return r.update_status === 'no-update' ? `${reach}; update: nothing to update${r.update_step ? ` (${clean(r.update_step)})` : ''}` : reach; }; const table = [ - '| Script | OS | Install | Update | Duration | Detail |', + '| Script | Platform | Install | Update | Duration | Detail |', '|---|---|---|---|---|---|', - ...rows.map(r => `| \`${r.script}\` | ${r.os} | ${r.update_status ? '✅' : icon[r.status] || '❔'} | ${updateIcon[r.update_status] || '–'} | ${dur(r.duration || 0)} | ${detail(r)} |`), + ...rows.map(r => `| \`${r.script}\` | ${where(r)} | ${r.update_status ? '✅' : icon[r.status] || '❔'} | ${updateIcon[r.update_status] || '–'} | ${dur(r.duration || 0)} | ${detail(r)} |`), ].join('\n'); // A comment holds 65536 characters. Each script gets an equal share @@ -510,25 +540,27 @@ jobs: const logs = rows.map(r => { const parts = []; const failed = r.status !== 'pass'; - const buildShare = failed && r.log_tail ? Math.min(r.log_tail.length, Math.floor(budget / 3)) : 0; - if (buildShare) parts.push(details(`${icon[r.status] || '❔'} ${r.script} (${r.os}) — build log`, r.log_tail, buildShare)); + // A VM script's own output is all there is: no install log inside. + const buildShare = (failed || isVm(r)) && r.log_tail ? Math.min(r.log_tail.length, Math.floor(budget / 3)) : 0; + if (buildShare) parts.push(details(`${icon[r.status] || '❔'} ${r.script} (${where(r)}) — build log`, r.log_tail, buildShare)); + if (isVm(r)) return parts.join('\n\n'); // The update log gets up to a third of what is left, the install log the rest. const updateShare = r.update_log ? Math.min(r.update_log.length, Math.floor((budget - buildShare) / 3)) : 0; if (r.install_log) { const lines = r.install_log.trimEnd().split('\n').length; - parts.push(details(`📄 ${r.script} (${r.os}) — install log (${lines} lines)`, r.install_log, budget - buildShare - updateShare - 400)); + parts.push(details(`📄 ${r.script} (${where(r)}) — install log (${lines} lines)`, r.install_log, budget - buildShare - updateShare - 400)); } else if (!buildShare) { - parts.push(`${r.script} (${r.os}): no install log was captured`); + parts.push(`${r.script} (${where(r)}): no install log was captured`); } if (updateShare) { const lines = r.update_log.trimEnd().split('\n').length; - parts.push(details(`🔄 ${r.script} (${r.os}) — update log (${lines} lines)`, r.update_log, updateShare)); + parts.push(details(`🔄 ${r.script} (${where(r)}) — update log (${lines} lines)`, r.update_log, updateShare)); } return parts.join('\n\n'); }); const shots = rows.filter(r => r.shot_url || r.shot_update_url).map(r => [ - `
📸 ${r.script} (${r.os}) — ${clean(r.url)}${r.shot_url && r.shot_update_url ? ': after install / after update' : r.shot_update_url ? ': after update' : ''}`, '', + `
📸 ${r.script} (${where(r)}) — ${r.url ? clean(r.url) : 'console'}${r.shot_url && r.shot_update_url ? ': after install / after update' : r.shot_update_url ? ': after update' : ''}`, '', [r.shot_url, r.shot_update_url].filter(Boolean) .map((u, i, all) => `Screenshot of ${r.script}`).join(' '), '', '
', @@ -638,7 +670,7 @@ jobs: owner, repo, pull_number: pr, commit_id: process.env.SHA, event: 'REQUEST_CHANGES', body: [ REVIEW_MARK, `**Proxmox script test failed** at \`${sha7}\`:`, '', - ...failedRows.map(r => `- \`${r.script}\` (${r.os}): ${clean(r.step)}`), '', + ...failedRows.map(r => `- \`${r.script}\` (${where(r)}): ${clean(r.step)}`), '', `Logs and details: [test result](${posted.html_url}). Push a fix, then ask for \`/test\` again.`, ].join('\n'), }); diff --git a/.github/workflows/scripts/incus-ci-host-setup.sh b/.github/workflows/scripts/incus-ci-host-setup.sh new file mode 100644 index 000000000..516e0a309 --- /dev/null +++ b/.github/workflows/scripts/incus-ci-host-setup.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# One-time setup of an Incus test host behind pr-script-test.yml: the same +# ct/ scripts as on Proxmox, built through community-scripts/core's Incus +# backend. Safe to re-run. Run as root on a disposable Debian 13 machine (it +# builds pull request code as root). +# +# incus-ci-host-setup.sh host only +# incus-ci-host-setup.sh host + a runner for that repo +# +# The token is a runner *registration* token (valid one hour): +# gh api -X POST repos//actions/runners/registration-token --jq .token + +set -euo pipefail + +RUNNER_VERSION="${RUNNER_VERSION:-2.338.2}" +SERVE_ROOT=/srv/ci +SERVE_PORT=18080 +NET_ADDR=10.98.0.1 +NET_CIDR=10.98.0.0/24 + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq incus jq curl python3 iptables chromium fonts-dejavu fonts-noto-color-emoji >/dev/null + +# Containers get a NATed bridge of their own (Incus runs DHCP and DNS on it) +# and a plain directory pool on the root disk. +if ! incus storage show default &>/dev/null; then + incus admin init --preseed </dev/null || useradd --system --create-home --shell /usr/sbin/nologin ci-shot + +# Same rules as on the Proxmox hosts: the screenshot browser may reach the +# test containers, DNS and the public internet, nothing private, and not this +# host's own address on the container network (the PR tree server). +cat >/usr/local/sbin/ci-shot-firewall </dev/null || \$fw -w -F CI_SHOT + \$fw -w -C OUTPUT -m owner --uid-owner "\$uid" -j CI_SHOT 2>/dev/null || + \$fw -w -I OUTPUT -m owner --uid-owner "\$uid" -j CI_SHOT +done +iptables -w -A CI_SHOT -d ${NET_ADDR} -j REJECT +iptables -w -A CI_SHOT -d ${NET_CIDR} -j ACCEPT +# systemd-resolved's stub; it forwards to the lab resolver itself. +iptables -w -A CI_SHOT -d 127.0.0.53 -p udp --dport 53 -j ACCEPT +iptables -w -A CI_SHOT -d 127.0.0.53 -p tcp --dport 53 -j ACCEPT +for ns in \$(awk '/^nameserver/ && \$2 !~ /:/ {print \$2}' /etc/resolv.conf /run/systemd/resolve/resolv.conf 2>/dev/null); do + iptables -w -A CI_SHOT -d "\$ns" -p udp --dport 53 -j ACCEPT + iptables -w -A CI_SHOT -d "\$ns" -p tcp --dport 53 -j ACCEPT +done +for net in 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 224.0.0.0/3; do + iptables -w -A CI_SHOT -d "\$net" -j REJECT +done +for net in ::1/128 fc00::/7 fe80::/10 ff00::/8; do + ip6tables -w -A CI_SHOT -d "\$net" -j REJECT +done +EOF +chmod +x /usr/local/sbin/ci-shot-firewall +cat >/etc/systemd/system/ci-shot-firewall.service <<'EOF' +[Unit] +Description=Keep the screenshot browser away from this host and the lab network +After=network-online.target incus.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/ci-shot-firewall + +[Install] +WantedBy=multi-user.target +EOF + +[[ -s /etc/timezone ]] || { + timedatectl set-timezone Europe/Vienna + echo Europe/Vienna >/etc/timezone +} + +# The rewritten PR trees, served to the engine and to the containers. Waits +# for incusbr0, which only exists once Incus is up. +mkdir -p "$SERVE_ROOT" +cat >/etc/systemd/system/ci-serve.service </dev/null 2>&1; sleep 1; done; exit 1' +ExecStart=/usr/bin/python3 -m http.server ${SERVE_PORT} --bind ${NET_ADDR} --directory ${SERVE_ROOT} +DynamicUser=yes +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +# Leftovers of a job that was killed before its cleanup trap ran. +cat >/usr/local/sbin/ci-sweep <<'EOF' +#!/usr/bin/env bash +# Delete CI containers (ci-9000 .. ci-9099) that no running job holds, and stale trees. +# A job holds its guest with /run/ci-jobs/. A job killed outright (runner +# restart, timeout) never removes it, so a lock older than any job can run +# (90 minutes) holds nothing. +held() { [[ -e "/run/ci-jobs/$1" && -z "$(find "/run/ci-jobs/$1" -mmin +180)" ]]; } +for name in $(incus list -c n -f csv 2>/dev/null | grep -E '^ci-9[0-9]{3}$'); do + held "$name" && continue + incus delete "$name" --force &>/dev/null && echo "swept $name" +done +find /run/ci-jobs -maxdepth 1 -type f ! -name ".*" -mmin +180 -delete 2>/dev/null +find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} + +find /usr/local/community-scripts/logs /var/log/community-scripts -type f -mtime +2 -delete 2>/dev/null +find /opt/actions-runner/*/_diag -type f -mtime +14 -delete 2>/dev/null +true +EOF +chmod +x /usr/local/sbin/ci-sweep +cat >/etc/systemd/system/ci-sweep.service <<'EOF' +[Unit] +Description=Remove leftover CI containers + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ci-sweep +EOF +cat >/etc/systemd/system/ci-sweep.timer <<'EOF' +[Unit] +Description=Remove leftover CI containers every 3 hours + +[Timer] +OnCalendar=0/3:30 +Persistent=true + +[Install] +WantedBy=timers.target +EOF + +systemctl daemon-reload +systemctl enable --now ci-serve.service ci-sweep.timer ci-shot-firewall.service >/dev/null +systemctl restart ci-serve.service ci-shot-firewall.service + +# --- Runner (one per repository) ---------------------------------------------- +if [[ $# -eq 2 ]]; then + repo="$1" + token="$2" + [[ "$repo" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad repo" >&2; exit 2; } + dir="/opt/actions-runner/${repo#*/}" + mkdir -p "$dir" + cd "$dir" + if [[ ! -x ./config.sh ]]; then + curl -fsSL "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -xz + ./bin/installdependencies.sh >/dev/null + fi + if [[ -f .runner ]]; then + ./svc.sh stop >/dev/null 2>&1 || true + ./svc.sh uninstall >/dev/null 2>&1 || true + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token "$token" >/dev/null 2>&1 || true + fi + # Only `incus-test`: see pve-ci-host-setup.sh for why there are no default labels. + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --unattended --replace --no-default-labels \ + --url "https://github.com/${repo}" --token "$token" \ + --name "$(hostname -s)-${repo#*/}" --labels incus-test --work _work + ./svc.sh install root >/dev/null + ./svc.sh start >/dev/null + echo "runner for ${repo} registered" +fi + +echo "incus test host ready: ${SERVE_ROOT} on ${NET_ADDR}:${SERVE_PORT}, sweeper every 3h" diff --git a/.github/workflows/scripts/pve-ci-host-setup.sh b/.github/workflows/scripts/pve-ci-host-setup.sh index 02193f784..43e7f4d2e 100644 --- a/.github/workflows/scripts/pve-ci-host-setup.sh +++ b/.github/workflows/scripts/pve-ci-host-setup.sh @@ -102,8 +102,9 @@ systemctl restart dnsmasq echo Europe/Vienna >/etc/timezone } -# Containers live on the root filesystem; there is no LVM in a cloud-image PVE. -pvesm set local --content iso,vztmpl,backup,rootdir,images +# Containers and VM disks live on the root filesystem; there is no LVM in a +# cloud-image PVE. Snippets hold the Cloud-Init configs some vm/ scripts write. +pvesm set local --content iso,vztmpl,backup,rootdir,images,snippets # Templates are fetched once here rather than on every first build. pveam update >/dev/null @@ -130,12 +131,26 @@ EOF # Leftovers of a job that was killed before its cleanup trap ran. cat >/usr/local/sbin/ci-sweep <<'EOF' #!/usr/bin/env bash -# Destroy CI containers (9000-9099) that no running job holds, and stale trees. +# Destroy CI containers (9000-9099) and VMs (9100-9199) that no running job +# holds, and stale trees. +# A job holds its guest with /run/ci-jobs/. A job killed outright (runner +# restart, timeout) never removes it, so a lock older than any job can run +# (90 minutes) holds nothing. +held() { [[ -e "/run/ci-jobs/$1" && -z "$(find "/run/ci-jobs/$1" -mmin +180)" ]]; } for id in $(pct list | awk 'NR>1 && $1>=9000 && $1<=9099 {print $1}'); do - [[ -e "/run/ci-jobs/$id" ]] && continue + held "$id" && continue pct stop "$id" --skiplock 1 &>/dev/null || true pct destroy "$id" --purge 1 --force 1 &>/dev/null && echo "swept CT $id" done +for id in $(qm list | awk 'NR>1 && $1>=9100 && $1<=9199 {print $1}'); do + held "$id" && continue + qm stop "$id" --skiplock 1 --timeout 30 &>/dev/null || true + qm destroy "$id" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null && echo "swept VM $id" +done +# VM images and ISOs are cached between builds; drop the ones no build used +# for a week. Container templates (*.tar.*) stay. +find /var/lib/vz/template/cache /var/lib/vz/template/iso -maxdepth 1 -type f ! -name '*.tar.*' -atime +7 -delete 2>/dev/null +find /run/ci-jobs -maxdepth 1 -type f ! -name ".*" -mmin +180 -delete 2>/dev/null find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} + # Engine logs that a killed job never got to remove. find /tmp -maxdepth 1 -type f \( -name 'create-lxc-*.log' -o -name 'pct_create_*.log' \ diff --git a/.github/workflows/scripts/pve-run-script.sh b/.github/workflows/scripts/pve-run-script.sh index e34cc426c..062f57b88 100644 --- a/.github/workflows/scripts/pve-run-script.sh +++ b/.github/workflows/scripts/pve-run-script.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash -# Build one ct/ script from a pull request on this Proxmox host, check that the -# container really works, destroy it, and leave a result.json behind. +# Build one ct/ or vm/ script from a pull request on this test host, check +# that the container or VM really works, destroy it, and leave a result.json +# behind. The vm/ part lives in pve-run-vm.sh, sourced from here. +# +# PLATFORM says what this host is: pve (Proxmox VE, pct) or incus. The ct/ +# scripts are the same for both; community-scripts/core picks its Incus +# backend by itself when it finds incus instead of pveversion. Only the way +# this harness reaches into the container differs (guest_* below). # # Runs on the self-hosted runner, which *is* the disposable test host. The # harness comes from the default branch; only the script under test comes from @@ -17,10 +23,17 @@ set -uo pipefail : "${PR_REPO:?}" "${PR_SHA:?}" "${SCRIPT:?}" "${UPSTREAM:?}" "${CTID_BASE:?}" "${OUT_DIR:?}" OS="${OS:-}" +PLATFORM="${PLATFORM:-pve}" CTID_COUNT="${CTID_COUNT:-50}" SERVE_ROOT="${SERVE_ROOT:-/srv/ci}" -SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}" -BRIDGE="${BRIDGE:-vmbr1}" +# The containers' own network: vmbr1 + dnsmasq on Proxmox, incusbr0 on Incus. +if [[ "$PLATFORM" == "incus" ]]; then + SERVE_URL="${SERVE_URL:-http://10.98.0.1:18080}" + BRIDGE="${BRIDGE:-incusbr0}" +else + SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}" + BRIDGE="${BRIDGE:-vmbr1}" +fi BUILD_TIMEOUT="${BUILD_TIMEOUT:-45m}" URL_WAIT="${URL_WAIT:-120}" @@ -28,8 +41,10 @@ URL_WAIT="${URL_WAIT:-120}" [[ "$PR_REPO" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad PR_REPO" >&2; exit 2; } [[ "$UPSTREAM" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad UPSTREAM" >&2; exit 2; } [[ "$PR_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "bad PR_SHA" >&2; exit 2; } -[[ "$SCRIPT" =~ ^ct/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; } +[[ "$SCRIPT" =~ ^(ct|vm)/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; } [[ -z "$OS" || "$OS" =~ ^(debian|ubuntu|alpine)$ ]] || { echo "bad OS" >&2; exit 2; } +[[ "$PLATFORM" =~ ^(pve|incus)$ ]] || { echo "bad PLATFORM" >&2; exit 2; } +[[ "$PLATFORM" == "pve" || "$SCRIPT" == ct/* ]] || { echo "vm/ scripts run on pve only" >&2; exit 2; } [[ "$CTID_BASE" =~ ^[0-9]+$ ]] || { echo "bad CTID_BASE" >&2; exit 2; } mkdir -p "$OUT_DIR" @@ -41,8 +56,9 @@ UPDATE_RAW="$OUT_DIR/update.raw.log" UPDATE_LOG="$OUT_DIR/update.log" UPDATE_TIMEOUT="${UPDATE_TIMEOUT:-30m}" +kind="${SCRIPT%%/*}" # ct or vm app=$(basename "$SCRIPT" .sh) -job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${app}-${OS:-default}" +job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${PLATFORM}-${kind}-${app}-${OS:-default}" tree="$SERVE_ROOT/$job" base="$SERVE_URL/$job" started=$(date +%s) @@ -50,7 +66,30 @@ started=$(date +%s) status="fail" step="" exit_code="" -ctid="" +ctid="" # the guest's ID (pve: 9000, incus: ci-9000), container or VM + +# The container, whatever the platform. ctid is set once a slot is reserved. +guest_exec() { + if [[ "$PLATFORM" == "incus" ]]; then incus exec "$ctid" -- "$@"; else pct exec "$ctid" -- "$@"; fi +} +guest_exists() { + if [[ "$PLATFORM" == "incus" ]]; then incus info "$ctid" &>/dev/null; else pct status "$ctid" &>/dev/null; fi +} +guest_running() { + if [[ "$PLATFORM" == "incus" ]]; then + [[ "$(incus list "^${ctid}\$" -c s -f csv 2>/dev/null)" == "RUNNING" ]] + else + pct status "$ctid" 2>/dev/null | grep -q running + fi +} +guest_destroy() { + if [[ "$PLATFORM" == "incus" ]]; then + incus delete "$ctid" --force &>/dev/null || true + else + pct stop "$ctid" --skiplock 1 &>/dev/null || true + pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true + fi +} ct_ip="" url="" http_code="" @@ -60,8 +99,13 @@ failed_units="" update_status="" update_step="" +# Spinner frames: the engine redraws "⠋ Starting VM" a few times a second +# without a tty too. An alternation, not a bracket: these are multibyte and +# the runner's locale may be C. +SPINNER_RE='^\s*(⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏)' + strip_ansi() { - sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' + sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' | grep -vE "$SPINNER_RE" } # The install script's own output: everything $STD hides from the build log. @@ -71,11 +115,12 @@ strip_ansi() { collect_install_log() { [[ -n "$ctid" ]] || return 0 # Exactly one build ran in this container, so the newest install log is ours. - if pct status "$ctid" &>/dev/null && - pct exec "$ctid" -- sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null | + if guest_exists && + guest_exec sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null | strip_ansi >"$INSTALL_LOG_OUT" && [[ -s "$INSTALL_LOG_OUT" ]]; then return 0 fi + [[ "$PLATFORM" == "pve" ]] || { rm -f "$INSTALL_LOG_OUT"; return 0; } # The engine names its host-side files after the session: pct_create___.log local sid f sid=$(find /tmp -maxdepth 1 -name "pct_create_${ctid}_*.log" -newer "$RAW_LOG.start" -printf '%f\n' 2>/dev/null | @@ -120,15 +165,15 @@ finish() { [[ -f "$UPDATE_RAW" ]] && strip_ansi <"$UPDATE_RAW" >"$UPDATE_LOG" rm -f "$UPDATE_RAW" jq -n \ - --arg script "$SCRIPT" --arg os "${OS:-default}" --arg status "$status" \ + --arg script "$SCRIPT" --arg kind "$kind" --arg platform "$PLATFORM" --arg os "${OS:-default}" --arg status "$status" \ --arg step "$step" --arg exit_code "$exit_code" --arg sha "$PR_SHA" \ --arg ctid "$ctid" --arg url "$url" --arg http_code "$http_code" \ - --arg failed_units "$failed_units" \ + --arg failed_units "$failed_units" --arg ip "$ct_ip" \ --arg update_status "$update_status" --arg update_step "$update_step" \ --argjson duration "$(($(date +%s) - started))" \ --arg tail "$([[ -f "$LOG" ]] && tail -n 60 "$LOG")" \ - '{script: $script, os: $os, status: $status, step: $step, exit_code: $exit_code, - sha: $sha, ctid: $ctid, url: $url, http_code: $http_code, + '{script: $script, kind: $kind, platform: $platform, os: $os, status: $status, step: $step, exit_code: $exit_code, + sha: $sha, ctid: $ctid, ip: $ip, url: $url, http_code: $http_code, failed_units: $failed_units, update_status: $update_status, update_step: $update_step, duration: $duration, log_tail: $tail}' >"$RESULT" echo "::group::result" @@ -150,7 +195,7 @@ install_pid="" # Colours stay, cursor movement and spinner redraws go. live_filter() { - sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E '/^\s*$/d' + sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E "/^\s*\$/d; /${SPINNER_RE}/d" } start_streams() { @@ -158,17 +203,21 @@ start_streams() { echo "::stop-commands::${stream_token}" tail -n +1 -F --pid="$build_pid" "$RAW_LOG" 2>/dev/null > >(live_filter) & log_pid=$! + # A VM has no install log to follow: its script runs on this host only. + [[ "$kind" == "ct" ]] || return 0 # setsid: its own process group, so stop_streams can take down the - # pct exec / tail pair inside it in one go. + # pct/incus exec + tail pair inside it in one go. setsid bash -c ' + gx() { if [ "$2" = incus ]; then incus exec "$1" -- "${@:3}"; else pct exec "$1" -- "${@:3}"; fi; } for _ in $(seq 1 900); do - f=$(pct exec "$1" -- sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null) + f=$(gx "$1" "$2" sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null) if [ -n "$f" ]; then echo "──────── install log of CT $1 (live) ────────" + if [ "$2" = incus ]; then exec incus exec "$1" -- tail -n +1 -F "$f"; fi exec pct exec "$1" -- tail -n +1 -F "$f" fi sleep 2 - done' _ "$ctid" 2>/dev/null > >(live_filter | sed -u 's/^/ │ /') & + done' _ "$ctid" "$PLATFORM" 2>/dev/null > >(live_filter | sed -u 's/^/ │ /') & install_pid=$! } @@ -195,15 +244,14 @@ cleanup() { kill -TERM "$pid" &>/dev/null || true sleep 2 done - if [[ -n "$ctid" ]]; then - if pct status "$ctid" &>/dev/null; then - pct stop "$ctid" --skiplock 1 &>/dev/null || true - pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true - fi + if [[ -n "$ctid" && "$kind" == "vm" ]]; then + vm_cleanup + elif [[ -n "$ctid" ]]; then + guest_exists && guest_destroy rm -f "/run/ci-jobs/$ctid" # What the engine leaves on the host for this container and its session. local f sid - for f in /tmp/pct_create_"${ctid}"_*.log; do + [[ "$PLATFORM" == "pve" ]] && for f in /tmp/pct_create_"${ctid}"_*.log; do [[ -f "$f" && "$f" -nt "$RAW_LOG.start" ]] || continue sid=$(sed -E 's/.*_([0-9a-f]+)\.log$/\1/' <<<"$f") rm -f -- "$f" /tmp/create-lxc-"${sid}".log /tmp/*-"${ctid}"-"${sid}".log @@ -245,19 +293,33 @@ setup_error() { # --- Reserve a container ID --------------------------------------------------- # A lock file per ID, not just "first free one", because the engine only creates # the container minutes later and two jobs would otherwise pick the same ID. +# VMs take the range 100 above the containers' (DevScripts 9100-9149, +# ProxmoxVE 9150-9199). mkdir -p /run/ci-jobs +if [[ "$kind" == "vm" ]]; then + source "$(dirname "$0")/pve-run-vm.sh" + CTID_BASE=$((CTID_BASE + 100)) + vm_wait_turn +fi exec 9>/run/ci-jobs/.lock flock 9 +# On Incus the ID is an instance name: var_ctid=ci-9000 names the container. for ((id = CTID_BASE; id < CTID_BASE + CTID_COUNT; id++)); do - [[ -e "/run/ci-jobs/$id" ]] && continue - pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue - echo "$job" >"/run/ci-jobs/$id" - ctid=$id + name=$id + [[ "$PLATFORM" == "incus" ]] && name="ci-$id" + [[ -e "/run/ci-jobs/$name" ]] && continue + if [[ "$PLATFORM" == "incus" ]]; then + incus info "$name" &>/dev/null && continue + else + pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue + fi + echo "$job" >"/run/ci-jobs/$name" + ctid=$name break done flock -u 9 exec 9>&- -[[ -n "$ctid" ]] || setup_error "no free container ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))" +[[ -n "$ctid" ]] || setup_error "no free ${kind} ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))" # --- Fetch the approved commit and point it at ourselves ------------------------ mkdir -p "$tree" @@ -269,17 +331,26 @@ upstream_re="https://raw\.githubusercontent\.com/${UPSTREAM//./\\.}/(refs/heads/ grep -rlE "$upstream_re" "$tree" --include='*.sh' --include='*.func' | xargs -r sed -i -E "s#${upstream_re}#${base}#g" +if [[ "$kind" == "vm" ]]; then + vm_build_and_check + exit 0 +fi + # --- Build ---------------------------------------------------------------------- -echo "Building ${SCRIPT} (${OS:-default OS}) as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" +echo "Building ${SCRIPT} (${OS:-default OS}) on ${PLATFORM} as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" hostname_safe="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-50)" build_env=( PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root TERM=dumb LANG=C.UTF-8 mode=default MODE=default PHS_SILENT=1 DIAGNOSTICS=no - var_ctid="$ctid" var_hostname="$hostname_safe" - var_container_storage=local var_template_storage=local var_brg="$BRIDGE" + var_ctid="$ctid" var_hostname="$hostname_safe" var_brg="$BRIDGE" "dev_mode=timing,keep" COMMUNITY_SCRIPTS_URL="$base" ) +if [[ "$PLATFORM" == "incus" ]]; then + build_env+=(var_container_storage=default) +else + build_env+=(var_container_storage=local var_template_storage=local) +fi touch "$RAW_LOG.start" [[ -n "$OS" ]] && build_env+=(var_os="$OS") # Both engines turn var_github_token into GITHUB_TOKEN inside the container. @@ -414,9 +485,9 @@ fi grep -qi 'completed successfully' "$RAW_LOG" || fail "build exited 0 but never reported success" # --- Checks --------------------------------------------------------------------- -pct status "$ctid" 2>/dev/null | grep -q running || fail "container ${ctid} is not running" +guest_running || fail "container ${ctid} is not running" -ct_ip=$(pct exec "$ctid" -- sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1) +ct_ip=$(guest_exec sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1) # What the app's page looks like, for the PR comment. Chromium runs as its own # unprivileged user with its sandbox on: the page is served by code under test. @@ -450,11 +521,11 @@ health_problem="" check_health() { health_problem="" failed_units="" - if pct exec "$ctid" -- sh -c 'command -v systemctl' &>/dev/null; then + if guest_exec sh -c 'command -v systemctl' &>/dev/null; then sleep 10 - failed_units=$(pct exec "$ctid" -- systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -) - elif pct exec "$ctid" -- sh -c 'command -v rc-status' &>/dev/null; then - failed_units=$(pct exec "$ctid" -- rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -) + failed_units=$(guest_exec systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -) + elif guest_exec sh -c 'command -v rc-status' &>/dev/null; then + failed_units=$(guest_exec rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -) fi [[ -z "$failed_units" ]] || { health_problem="failed services: ${failed_units}"; return 1; } [[ -n "$url" ]] || return 0 @@ -486,14 +557,14 @@ update_fail() { fail "update: $1" } -version_files=$(pct exec "$ctid" -- sh -c ' +version_files=$(guest_exec sh -c ' for f in /root/.*; do [ -f "$f" ] || continue case "$f" in */.install-* | *.pending | *.bak | */.bash* | */.profile | */.hushlogin | */.viminfo | */.lesshst | */.wget-hsts | */.selected_editor) continue ;; esac [ "$(wc -l <"$f")" -le 1 ] && grep -qE "^v?[0-9]+(\.[0-9A-Za-z_+-]+)+$" "$f" && echo "$f" done' 2>/dev/null | grep -E '^/root/\.[A-Za-z0-9._-]+$') for f in $version_files; do - pct exec "$ctid" -- sh -c "echo '$UPDATE_FAKE' > '$f'; rm -f '$f.pending'" || + guest_exec sh -c "echo '$UPDATE_FAKE' > '$f'; rm -f '$f.pending'" || update_fail "could not reset $f" done [[ -n "$version_files" ]] && echo "Update test: set back $(tr '\n' ' ' <<<"$version_files")to ${UPDATE_FAKE}" @@ -502,7 +573,9 @@ done update_token=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n') echo "──────── update test (live) ────────" echo "::stop-commands::${update_token}" -timeout --kill-after=1m "$UPDATE_TIMEOUT" pct exec "$ctid" -- env PHS_SILENT=1 mode=default MODE=default \ +# timeout runs a program, not a shell function, so spell out the exec here. +if [[ "$PLATFORM" == "incus" ]]; then guest_cmd=(incus exec "$ctid" --); else guest_cmd=(pct exec "$ctid" --); fi +timeout --kill-after=1m "$UPDATE_TIMEOUT" "${guest_cmd[@]}" env PHS_SILENT=1 mode=default MODE=default \ TERM=dumb DIAGNOSTICS=no var_ignore_breaking_changes=yes ${GH_API_TOKEN:+GITHUB_TOKEN="$GH_API_TOKEN"} update >(tee "$UPDATE_RAW" | live_filter) 2>&1 & update_pid=$! @@ -538,7 +611,7 @@ fi if strip_ansi <"$UPDATE_RAW" | grep -qi 'updated successfully'; then update_status="updated" for f in $version_files; do - [[ "$(pct exec "$ctid" -- cat "$f" 2>/dev/null)" != "$UPDATE_FAKE" ]] || + [[ "$(guest_exec cat "$f" 2>/dev/null)" != "$UPDATE_FAKE" ]] || update_fail "reported success, but $f is still ${UPDATE_FAKE}" done else diff --git a/.github/workflows/scripts/pve-run-vm.sh b/.github/workflows/scripts/pve-run-vm.sh new file mode 100644 index 000000000..5492566de --- /dev/null +++ b/.github/workflows/scripts/pve-run-vm.sh @@ -0,0 +1,140 @@ +# shellcheck shell=bash +# The vm/ half of pve-run-script.sh, sourced from it: same inputs, same +# result.json, same live output. Only building and checking differ. +# +# A VM script runs entirely on this host (there is no install script inside +# the guest to follow), unattended through the core engine's VM_UNATTENDED=1: +# the script's defaults, with ID, name, bridge and storage set from here. The +# guest then boots on vmbr1 like the containers do, so its address is whatever +# this host's dnsmasq leased to its MAC; no guest agent needed. +# +# What passes: the script exits 0 and says "Completed successfully", the VM is +# running and keeps running through the boot wait, and where the script prints +# a URL built from the VM's address (http://${VM_IP}:9000) that URL answers 200. +# An installer ISO or an image without DHCP gets no address; that is reported, +# not failed. There is no update test: a VM has no `update`. + +VM_BOOT_WAIT="${VM_BOOT_WAIT:-300}" +VM_URL_WAIT="${VM_URL_WAIT:-600}" +LEASES="${LEASES:-/var/lib/misc/dnsmasq.leases}" + +# One VM build per host at a time: a VM takes 2-8 GB of RAM and a disk image, +# where a container takes a fraction. Held until the job exits; the build is +# started with this descriptor closed, so a QEMU process can never inherit it. +vm_wait_turn() { + exec 7>/run/ci-jobs/.vm-turn + flock -n 7 && return 0 + echo "Another VM build is running on this host; waiting for it (one VM at a time)" + flock -w 3600 7 || setup_error "waited an hour for this host's VM slot" +} + +vm_cleanup() { + if qm status "$ctid" &>/dev/null; then + qm stop "$ctid" --skiplock 1 --timeout 30 &>/dev/null || true + qm destroy "$ctid" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null || true + fi + rm -f "/run/ci-jobs/$ctid" "/run/ci-jobs/console-${ctid}.png" + # Cloud-Init network snippets the script wrote for this VM. + [[ -f "$RAW_LOG.start" ]] && + find /var/lib/vz/snippets -maxdepth 1 -type f -name "*${ctid}*" -newer "$RAW_LOG.start" -delete 2>/dev/null + true +} + +# The VM's screen, straight from QEMU. Pixels only, but they come from the +# guest, so the same checks as for a browser screenshot apply. +vm_console_shot() { + local f="/run/ci-jobs/console-${ctid}.png" + rm -f "$f" + printf 'screendump %s -f png\n' "$f" | timeout 30 qm monitor "$ctid" &>/dev/null + if [[ -s "$f" && $(stat -c %s "$f") -lt 3000000 ]] && + [[ "$(head -c 8 "$f" | od -An -tx1 | tr -d ' \n')" == "89504e470d0a1a0a" ]]; then + cp "$f" "$OUT_DIR/screenshot.png" + fi + rm -f "$f" +} + +vm_running() { + qm status "$ctid" 2>/dev/null | grep -q 'status: running' +} + +vm_build_and_check() { + local hn mac pattern waited last + hn="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-40)-${ctid}" + echo "Building ${SCRIPT} as VM ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" + local vm_env=( + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + HOME=/root TERM=dumb LANG=C.UTF-8 DIAGNOSTICS=no + VM_UNATTENDED=1 VM_VMID="$ctid" VM_HOSTNAME="$hn" VM_BRIDGE="$BRIDGE" + VM_STORAGE=local VM_ISO_STORAGE=local VM_START=yes + COMMUNITY_SCRIPTS_URL="$base" + ) + # Release lookups (vm_latest_from_github and the like). + [[ -n "${GH_API_TOKEN:-}" ]] && vm_env+=(GITHUB_TOKEN="$GH_API_TOKEN") + touch "$RAW_LOG.start" + + env -i "${vm_env[@]}" timeout --kill-after=1m "$BUILD_TIMEOUT" \ + bash -c 'bash -c "$(curl -fsSL "$1")"' _ "$base/$SCRIPT" "$RAW_LOG" 2>&1 7>&- & + build_pid=$! + start_streams + wait "$build_pid" + exit_code=$? + build_pid="" + stop_streams + + ((exit_code == 124 || exit_code == 137)) && fail "build timed out after ${BUILD_TIMEOUT}" + if ((exit_code != 0)); then + last=$(failure_line) + step="script exited ${exit_code}${last:+: $last}" + status="fail" + classify_cpu_failure + fail "$step" "$status" + fi + # vm_finish ends on a msg_ok of the script's choosing ("VM created. ..."), + # older scripts on "Completed successfully". exit_script and friends leave + # with 0 too, but on a cross. + last=$(strip_ansi <"$RAW_LOG" | tail -n 1 | cut -c1-200) + if ! grep -qi 'completed successfully' "$RAW_LOG" && ! [[ "$last" =~ ^[[:space:]]*✔ ]]; then + fail "script exited 0 but never reported success${last:+ (last line: ${last})}" + fi + vm_running || fail "VM ${ctid} is not running after the script finished" + + # The address dnsmasq handed to this VM's MAC. The VM must stay up while it + # boots: one that powers itself off or crashes is a failure. + mac=$(qm config "$ctid" 2>/dev/null | sed -nE 's/^net0:[^=]*=([0-9A-Fa-f:]{17}).*/\1/p' | tr 'A-F' 'a-f') + echo "Waiting up to ${VM_BOOT_WAIT}s for VM ${ctid} (${mac:-no NIC}) to boot and get an address" + for ((waited = 0; waited < VM_BOOT_WAIT; waited += 5)); do + vm_running || fail "VM ${ctid} stopped by itself while booting" + [[ -n "$mac" ]] && ct_ip=$(awk -v m="$mac" '$2 == m {print $3}' "$LEASES" 2>/dev/null | tail -n 1) + [[ -n "$ct_ip" ]] && break + sleep 5 + done + [[ -n "$ct_ip" ]] && echo "VM ${ctid} got ${ct_ip}" || echo "VM ${ctid} got no DHCP lease within ${VM_BOOT_WAIT}s" + + # Only a URL the script builds from the address it found; in a + # summary is a hint for later (after an installer ran), not a promise. + # ${VM_IP}, $VM_IP and ${VM_IP:-} all count. + local ip_re='\$\{?VM_IP(:-[^}]*)?\}?' + pattern=$(grep -oE "https?://${ip_re}(:[0-9]+)?(/[A-Za-z0-9._~/?=&%-]*)?" "$tree/$SCRIPT" | head -n1) + if [[ -n "$pattern" ]]; then + [[ -n "$ct_ip" ]] || fail "no IP address within ${VM_BOOT_WAIT}s, so ${pattern} could not be checked" + url=$(sed -E "s#${ip_re}#${ct_ip}#" <<<"$pattern") + for ((waited = 0; waited < VM_URL_WAIT; waited += 10)); do + vm_running || fail "VM ${ctid} stopped by itself while its web UI was awaited" + http_code=$(curl -sk -L --max-redirs 5 -o /dev/null -w '%{http_code}' --max-time 10 "$url" || true) + [[ "$http_code" =~ ^2[0-9][0-9]$ ]] && break + sleep 10 + done + [[ "$http_code" =~ ^[1-5][0-9][0-9]$ ]] && take_screenshot screenshot.png + else + # Let the console settle on a login prompt rather than a boot message. + sleep 20 + fi + [[ -s "$OUT_DIR/screenshot.png" ]] || vm_console_shot + if [[ -n "$url" && ! "$http_code" =~ ^2[0-9][0-9]$ ]]; then + fail "${url} answered ${http_code/#000/nothing} instead of 200 within ${VM_URL_WAIT}s" + fi + vm_running || fail "VM ${ctid} stopped by itself" + + status="pass" + finish +}