#!/usr/bin/env node // Compare two tasty-bench CSV files, plotting change in time and allocation // per benchmark on the same SVG chart. // // Usage: node benchmark-alloc.js const fs = require('fs'); const [oldPath, newPath] = process.argv.slice(2); if (!oldPath || !newPath) { console.error('Usage: node benchmark-alloc.js '); process.exit(1); } // Derive short labels from filenames, e.g. "benchmark-3.10.2.csv" -> "3.10.2" const shortName = p => p.replace(/^.*\//, '').replace(/^(benchmark-|bench_)/, '').replace(/\.csv$/, ''); const oldLabel = shortName(oldPath); const newLabel = shortName(newPath); function parseCsv(path) { const lines = fs.readFileSync(path, 'utf8').trim().split('\n'); const out = new Map(); for (const line of lines.slice(1)) { const cols = line.split(','); out.set(cols[0], { time: Number(cols[1]), alloc: Number(cols[3]) }); } return out; } const oldData = parseCsv(oldPath); const newData = parseCsv(newPath); function fmtMs(ps) { const ms = ps / 1e9; if (ms >= 100) return ms.toFixed(0) + ' ms'; if (ms >= 10) return ms.toFixed(1) + ' ms'; return ms.toFixed(2) + ' ms'; } function fmtBytes(b) { const mb = b / (1024 * 1024); if (mb >= 100) return mb.toFixed(0) + ' MB'; if (mb >= 10) return mb.toFixed(1) + ' MB'; if (mb >= 1) return mb.toFixed(2) + ' MB'; return (b / 1024).toFixed(0) + ' KB'; } // Collect rows per group const groups = { writers: [], readers: [] }; for (const [name, o] of oldData) { const m = name.match(/^All\.(writers|readers)\.(.+)$/); if (!m) continue; const n = newData.get(name); if (!n) continue; groups[m[1]].push({ label: m[2], oldTime: o.time, newTime: n.time, oldAlloc: o.alloc, newAlloc: n.alloc, timeChange: (n.time - o.time) / o.time, allocChange: (n.alloc - o.alloc) / o.alloc, }); } // New benchmarks with no baseline const added = []; for (const [name] of newData) { if (!oldData.has(name)) added.push(name.replace(/^All\./, '')); } for (const g of Object.values(groups)) g.sort((a, b) => a.timeChange - b.timeChange); // Colors: time = green, alloc = red (as requested) const TIME_COLOR = '#2f9e44'; const ALLOC_COLOR = '#d9480f'; // Layout const rowH = 24; const labelW = 175; const chartW = 430; const rightW = 260; const margin = 22; const width = margin + labelW + chartW + rightW + margin; const headerH = 100; const sectionGap = 40; const footerH = added.length ? 46 : 24; const allRows = [...groups.writers, ...groups.readers]; const allChanges = allRows.flatMap(r => [r.timeChange, r.allocChange]); const minC = Math.min(...allChanges); const maxC = Math.max(...allChanges); // domain with padding, always include 0 const lo = Math.min(0, minC) * 1.06; const hi = Math.max(0.05, maxC * 1.4); const x = c => margin + labelW + ((c - lo) / (hi - lo)) * chartW; const sections = [ ['Writers', groups.writers], ['Readers', groups.readers], ]; let height = headerH; for (const [, rows] of sections) height += 34 + rows.length * rowH + sectionGap; height += footerH - sectionGap; const esc = s => s.replace(/&/g, '&').replace(/`); svg.push(``); svg.push(`pandoc benchmarks: ${esc(oldLabel)} \u2192 ${esc(newLabel)}`); svg.push(`Change in mean wall-clock time and allocated bytes per benchmark (tasty-bench). Negative = better. Sorted by time.`); // Legend const legY = 76; svg.push(``); svg.push(`time`); svg.push(``); svg.push(`allocation`); let y = headerH; for (const [title, rows] of sections) { svg.push(`${title}`); y += 24; const top = y, bottom = y + rows.length * rowH; // gridlines + axis ticks const step = 0.25; for (let t = Math.ceil(lo / step) * step; t <= hi + 1e-9; t += step) { const tx = x(t); const zero = Math.abs(t) < 1e-9; svg.push(``); svg.push(`${(t * 100).toFixed(0)}%`); } for (const r of rows) { const barH = 8, barGap = 2; const x0 = x(0); // one bar per metric: time first (top, green), then alloc (bottom, red) const bars = [ { change: r.timeChange, color: TIME_COLOR, cy: y + rowH / 2 - (barH + barGap) / 2 }, { change: r.allocChange, color: ALLOC_COLOR, cy: y + rowH / 2 + (barH + barGap) / 2 }, ]; for (const b of bars) { const x1 = x(b.change); const neg = b.change < 0; svg.push(``); const pct = (b.change * 100).toFixed(Math.abs(b.change) < 0.095 ? 1 : 0) + '%'; const pctTxt = (b.change > 0 ? '+' : '') + pct; const tipX = neg ? x1 - 4 : x1 + 4; svg.push(`${pctTxt}`); } // benchmark label svg.push(`${esc(r.label)}`); // absolute values on right: time on top line, alloc below const rx = margin + labelW + chartW + 14; svg.push(`${fmtMs(r.oldTime)} \u2192 ${fmtMs(r.newTime)}`); svg.push(`${fmtBytes(r.oldAlloc)} \u2192 ${fmtBytes(r.newAlloc)}`); y += rowH; } y += sectionGap; } if (added.length) { svg.push(`New benchmarks (no ${esc(oldLabel)} baseline): ${esc(added.join(', '))}`); } svg.push(`Mean times and allocated bytes from tasty-bench CSV output; % change = (${esc(newLabel)} \u2212 ${esc(oldLabel)}) / ${esc(oldLabel)}.`); svg.push(''); const outFile = 'benchmark-comparison.svg'; fs.writeFileSync(outFile, svg.join('\n')); console.log(`Wrote ${outFile} (${width}x${height})`); console.log(`writers: ${groups.writers.length} rows, readers: ${groups.readers.length} rows`); console.log(`time change range: ${(Math.min(...allRows.map(r => r.timeChange)) * 100).toFixed(1)}% .. ${(Math.max(...allRows.map(r => r.timeChange)) * 100).toFixed(1)}%`); console.log(`alloc change range: ${(Math.min(...allRows.map(r => r.allocChange)) * 100).toFixed(1)}% .. ${(Math.max(...allRows.map(r => r.allocChange)) * 100).toFixed(1)}%`);