update
ober
71ec82bed4c5099029c1aae62ba994e73ae5f6f7
new file mode 100644 --- /dev/null +++ b/support/wasm-gc/abi-manifest.mjs @@ -0,0 +1,78 @@ +const CORE_IMPORTS = Object.freeze([ + Object.freeze({ module: "jerboa", name: "display_i32", kind: "function" }), + Object.freeze({ module: "jerboa", name: "extern_value", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "undefined", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "number", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "string", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "global", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "get", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "set", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "call0", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "call1", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "call2", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "call3", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "invoke0", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "invoke1", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "invoke2", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "invoke3", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "new0", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "new1", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "new2", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "new3", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "function0", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "function1", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "function2", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "function3", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "to_number", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "string_length", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "string_char", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "equal", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "truthy", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "number_f64", kind: "function" }), + Object.freeze({ module: "jerboa.js", name: "to_f64", kind: "function" }), + Object.freeze({ module: "jerboa.host_exception", name: "pending", kind: "function" }), + Object.freeze({ module: "jerboa.host_exception", name: "message_length", kind: "function" }), + Object.freeze({ module: "jerboa.host_exception", name: "message_char", kind: "function" }), + Object.freeze({ module: "jerboa.host_exception", name: "clear", kind: "function" }), + Object.freeze({ module: "jerboa.gfx", name: "replay", kind: "function" }), + Object.freeze({ module: "jerboa", name: "display_string", kind: "function" }), + Object.freeze({ module: "jerboa", name: "display_i64", kind: "function" }), +]); + +const CORE_FEATURES = Object.freeze(["gc", "tail-call", "typed-funcref", "externref", "js-bridge"]); + +const CORE_PROFILE = Object.freeze({ + features: CORE_FEATURES, + imports: CORE_IMPORTS, +}); + +export const JERBOA_WASM_GC_ABI = Object.freeze({ + backend: "wasm-gc", + abi: "1", + compiler: "jerboa-wasm-gc-stage0", + profiles: Object.freeze({ + core: CORE_PROFILE, + node: CORE_PROFILE, + browser: CORE_PROFILE, + }), +}); + +export function importKey(desc) { + return `${desc.module}.${desc.name}:${desc.kind}`; +} + +export function expectedImportsForProfile(profile = "core") { + const manifest = JERBOA_WASM_GC_ABI.profiles[profile]; + if (!manifest) { + throw new Error(`unknown Jerboa Wasm-GC profile ${profile}`); + } + return [...manifest.imports]; +} + +export function featuresForProfile(profile = "core") { + const manifest = JERBOA_WASM_GC_ABI.profiles[profile]; + if (!manifest) { + throw new Error(`unknown Jerboa Wasm-GC profile ${profile}`); + } + return [...manifest.features]; +} new file mode 100644 --- /dev/null +++ b/support/wasm-gc/benchmark-report.mjs @@ -0,0 +1,754 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { createServer } from "node:http"; +import { existsSync, readdirSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { spawnSync } from "node:child_process"; +import { + assertAllowedJerboaImports, + assertSupportedJerboaAbi, + makeJerboaImports, +} from "./host.mjs"; +import { + BrowserEngineUnavailable, + SUPPORTED_BROWSER_ENGINES, + runBrowserResultPage, +} from "./browser-engine.mjs"; + +const REPORT_SCHEMA = "jerboa.wasm-gc.benchmark-report.v1"; +const SUPPORTED_ENGINES = Object.freeze(["node", ...SUPPORTED_BROWSER_ENGINES]); +const repoRoot = resolve(new URL("../..", import.meta.url).pathname); + +function usage() { + console.error("usage: benchmark-report.mjs [--quick] [--check-manifest] [--engine node[,chromium,firefox,webkit,safari]] [--allow-missing-engine] manifest.json [report.json]"); +} + +function parseArgs(argv) { + const options = { + quick: false, + checkManifest: false, + allowMissingEngine: false, + engines: ["node"], + positionals: [], + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--quick") { + options.quick = true; + } else if (arg === "--check-manifest") { + options.checkManifest = true; + } else if (arg === "--allow-missing-engine") { + options.allowMissingEngine = true; + } else if (arg === "--engine") { + i += 1; + if (i >= argv.length) throw new Error("--engine requires an engine list"); + options.engines = argv[i].split(",").filter((engine) => engine.length > 0); + } else if (arg.startsWith("-")) { + throw new Error(`unknown option ${arg}`); + } else { + options.positionals.push(arg); + } + } + if (options.positionals.length < 1 || options.positionals.length > 2) { + usage(); + process.exit(2); + } + for (const engine of options.engines) { + if (!SUPPORTED_ENGINES.includes(engine)) { + throw new Error(`unsupported benchmark engine ${engine}`); + } + } + return options; +} + +function validateManifest(manifest) { + if (manifest.schema !== "jerboa.wasm-gc.benchmark-manifest.v1") { + throw new Error("unsupported Wasm-GC benchmark manifest schema"); + } + if (!Array.isArray(manifest.cases) || manifest.cases.length === 0) { + throw new Error("benchmark manifest requires at least one case"); + } + for (const bench of manifest.cases) { + if (!bench.id || !/^[A-Za-z0-9_.:-]+$/.test(bench.id)) { + throw new Error(`invalid benchmark id ${bench.id}`); + } + if (typeof bench.source !== "string" || bench.source.length === 0) { + throw new Error(`${bench.id}: source is required`); + } + if (typeof bench.export !== "string" || bench.export.length === 0) { + throw new Error(`${bench.id}: export is required`); + } + if (!Array.isArray(bench.args)) { + throw new Error(`${bench.id}: args must be an array`); + } + if (!Array.isArray(bench.targets) || bench.targets.length === 0) { + throw new Error(`${bench.id}: targets must be a non-empty array`); + } + if (!Array.isArray(bench.optLevels) || bench.optLevels.length === 0) { + throw new Error(`${bench.id}: optLevels must be a non-empty array`); + } + if (bench.budgets !== undefined) { + if (!bench.budgets || typeof bench.budgets !== "object" || Array.isArray(bench.budgets)) { + throw new Error(`${bench.id}: budgets must be an object`); + } + for (const [name, value] of Object.entries(bench.budgets)) { + if (value !== null && !(typeof value === "number" && Number.isFinite(value) && value >= 0)) { + throw new Error(`${bench.id}: budget ${name} must be a non-negative number or null`); + } + } + } + } +} + +function runCommand(command, args, options = {}) { + return new Promise((resolvePromise, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code, signal) => { + if (code === 0) { + resolvePromise({ stdout, stderr }); + } else { + reject(new Error(`${command} ${args.join(" ")} failed with ${signal ?? code}\n${stderr || stdout}`)); + } + }); + }); +} + +function nsSince(start) { + return Number(process.hrtime.bigint() - start); +} + +function chromePath() { + if (process.env.CHROME_PATH) return process.env.CHROME_PATH; + const absoluteCandidates = [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ]; + for (const candidate of absoluteCandidates) { + if (existsSync(candidate)) return candidate; + } + const pathCandidates = [ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + ]; + for (const candidate of pathCandidates) { + const probe = spawnSync("which", [candidate], { encoding: "utf8" }); + if (probe.status === 0) return probe.stdout.trim(); + } + return null; +} + +function contentType(pathname) { + if (pathname.endsWith(".mjs")) return "text/javascript; charset=utf-8"; + if (pathname.endsWith(".wasm")) return "application/wasm"; + return "text/html; charset=utf-8"; +} + +function delay(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +function waitForDebuggerURL(child) { + return new Promise((resolvePromise, reject) => { + let done = false; + let stderr = ""; + const timer = setTimeout(() => { + if (done) return; + done = true; + reject(new Error(stderr || "Chromium did not expose a DevTools endpoint")); + }, 10000); + child.stderr.on("data", (chunk) => { + if (done) return; + stderr += String(chunk); + const match = stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/); + if (match) { + done = true; + clearTimeout(timer); + resolvePromise(match[1]); + } + }); + child.on("error", (err) => { + if (done) return; + done = true; + clearTimeout(timer); + reject(err); + }); + child.on("exit", (code) => { + if (done) return; + done = true; + clearTimeout(timer); + reject(new Error(stderr || `Chromium exited before DevTools was ready: ${code}`)); + }); + }); +} + +async function pageWebSocketURL(browserURL, pageURL) { + const endpoint = new URL(browserURL); + const listURL = `http://${endpoint.host}/json/list`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const response = await fetch(listURL).catch(() => null); + if (response?.ok) { + const targets = await response.json(); + const page = targets.find((target) => + target.type === "page" && target.url === pageURL && target.webSocketDebuggerUrl + ); + if (page) return page.webSocketDebuggerUrl; + } + await delay(100); + } + throw new Error("Chromium page target was not available through DevTools"); +} + +function connectWebSocket(url) { + return new Promise((resolvePromise, reject) => { + const ws = new WebSocket(url); + ws.addEventListener("open", () => resolvePromise(ws), { once: true }); + ws.addEventListener("error", reject, { once: true }); + }); +} + +function terminateProcess(child) { + return new Promise((resolvePromise) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolvePromise(); + return; + } + child.once("close", () => resolvePromise()); + child.kill(); + setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, 3000).unref(); + }); +} + +function cdpCommand(ws, method, params = {}) { + const id = cdpCommand.nextId++; + ws.send(JSON.stringify({ id, method, params })); + return new Promise((resolvePromise, reject) => { + function onMessage(event) { + const message = JSON.parse(String(event.data)); + if (message.id !== id) return; + ws.removeEventListener("message", onMessage); + if (message.error) { + reject(new Error(message.error.message || JSON.stringify(message.error))); + } else { + resolvePromise(message.result); + } + } + ws.addEventListener("message", onMessage); + }); +} +cdpCommand.nextId = 1; + +async function evaluate(ws, expression) { + const result = await cdpCommand(ws, "Runtime.evaluate", { + expression, + returnByValue: true, + awaitPromise: true, + }); + if (result.exceptionDetails) { + throw new Error(result.exceptionDetails.text || "browser evaluation failed"); + } + return result.result.value; +} + +async function instantiateMeasuredNode(module, profile) { + const abi = assertSupportedJerboaAbi(module); + const activeProfile = abi ? abi.profile : profile; + assertAllowedJerboaImports(module, { profile: activeProfile }); + const bridgeState = {}; + const imports = makeJerboaImports({ + profile: activeProfile, + displayI32() {}, + displayI64() {}, + displayString() {}, + }, bridgeState); + const start = process.hrtime.bigint(); + const instance = await WebAssembly.instantiate(module, imports); + const instantiateNs = nsSince(start); + bridgeState.instance = instance; + bridgeState.memory = instance.exports.memory; + return { instance, instantiateNs, profile: activeProfile }; +} + +async function measureNode(bytes, bench, target, iterations) { + const validateStart = process.hrtime.bigint(); + const valid = WebAssembly.validate(bytes); + const validateNs = nsSince(validateStart); + if (!valid) { + throw new Error(`${bench.id}: WebAssembly.validate returned false`); + } + const engineCompileStart = process.hrtime.bigint(); + const module = await WebAssembly.compile(bytes); + const compileNs = nsSince(engineCompileStart); + const { instance, instantiateNs, profile } = await instantiateMeasuredNode(module, target); + const fn = instance.exports[bench.export]; + if (typeof fn !== "function") { + throw new Error(`${bench.id}: missing export ${bench.export}`); + } + + let lastResult; + const runStart = process.hrtime.bigint(); + for (let i = 0; i < iterations; i += 1) { + lastResult = fn(...bench.args); + } + const runTotalNs = nsSince(runStart); + return { + engineName: "node", + target: profile, + result: String(lastResult), + metrics: { + validateNs, + compileNs, + instantiateNs, + runTotalNs, + runMeanNs: Math.round(runTotalNs / iterations), + }, + }; +} + +function benchmarkPage(bench, iterations, engineName) { + return `<!doctype html> +<meta charset="utf-8"> +<body data-ok="pending">pending</body> +<script type="module"> +import { + assertAllowedJerboaImports, + assertSupportedJerboaAbi, + makeJerboaImports, +} from "/support/wasm-gc/host.mjs"; +async function finish(ok, payload) { + document.body.dataset.ok = ok ? "true" : "false"; + document.body.textContent = JSON.stringify(payload); + await fetch("/result", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok, payload }), + }).catch(() => undefined); +} +try { + const response = await fetch("/module.wasm"); + if (!response.ok) throw new Error("module fetch failed: HTTP " + response.status); + const bytes = new Uint8Array(await response.arrayBuffer()); + const validateStart = performance.now(); + const valid = WebAssembly.validate(bytes); + const validateNs = Math.round((performance.now() - validateStart) * 1000000); + if (!valid) throw new Error("WebAssembly.validate returned false"); + const compileStart = performance.now(); + const module = await WebAssembly.compile(bytes); + const compileNs = Math.round((performance.now() - compileStart) * 1000000); + const abi = assertSupportedJerboaAbi(module); + const profile = abi ? abi.profile : ${JSON.stringify("core")}; + assertAllowedJerboaImports(module, { profile }); + const bridgeState = {}; + const imports = makeJerboaImports({ + profile, + displayI32() {}, + displayI64() {}, + displayString() {}, + }, bridgeState); + const instantiateStart = performance.now(); + const instance = await WebAssembly.instantiate(module, imports); + const instantiateNs = Math.round((performance.now() - instantiateStart) * 1000000); + bridgeState.instance = instance; + bridgeState.memory = instance.exports.memory; + const fn = instance.exports[${JSON.stringify(bench.export)}]; + if (typeof fn !== "function") throw new Error(${JSON.stringify("missing export " + bench.export)}); + const args = ${JSON.stringify(bench.args)}; + const iterations = ${JSON.stringify(iterations)}; + let result; + const runStart = performance.now(); + for (let i = 0; i < iterations; i += 1) { + result = fn(...args); + } + const runTotalNs = Math.round((performance.now() - runStart) * 1000000); + await finish(true, { + engineName: ${JSON.stringify(engineName)}, + target: profile, + result: String(result), + metrics: { + validateNs, + compileNs, + instantiateNs, + runTotalNs, + runMeanNs: Math.round(runTotalNs / iterations), + }, + }); +} catch (err) { + await finish(false, { + error: err && err.message ? err.message : String(err), + }); +} +</script>`; +} + +function latestPlaywrightWebKitRunner() { + if (process.env.PLAYWRIGHT_WEBKIT_RUNNER) return process.env.PLAYWRIGHT_WEBKIT_RUNNER; + const root = process.env.PLAYWRIGHT_BROWSERS_PATH || + (process.env.HOME ? join(process.env.HOME, "Library/Caches/ms-playwright") : null); + if (!root || !existsSync(root)) return null; + const candidates = readdirSync(root) + .filter((name) => /^webkit-\d+$/u.test(name)) + .sort((left, right) => Number(right.slice(7)) - Number(left.slice(7))) + .map((name) => join(root, name, "pw_run.sh")); + return candidates.find((candidate) => existsSync(candidate)) || null; +} + +function spawnWebKitPage(pageURL) { + const runner = latestPlaywrightWebKitRunner(); + if (!runner) { + throw new BrowserEngineUnavailable( + "webkit", + "Playwright WebKit not available; run `npx playwright install webkit` or set PLAYWRIGHT_WEBKIT_RUNNER" + ); + } + const webkitDir = dirname(runner); + const executable = join(webkitDir, "Playwright.app/Contents/MacOS/Playwright"); + if (!existsSync(executable)) { + throw new BrowserEngineUnavailable("webkit", `Playwright WebKit executable not found: ${executable}`); + } + const child = spawn(executable, ["--headless", pageURL], { + env: { + ...process.env, + DYLD_FRAMEWORK_PATH: webkitDir, + DYLD_LIBRARY_PATH: process.env.DYLD_LIBRARY_PATH + ? `${webkitDir}:${process.env.DYLD_LIBRARY_PATH}` + : webkitDir, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (chunk) => { + output = `${output}${String(chunk)}`.slice(-4000); + }); + child.stderr.on("data", (chunk) => { + output = `${output}${String(chunk)}`.slice(-4000); + }); + return { child, output: () => output.trim() }; +} + +function withTimeout(promise, ms, message) { + let timer; + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), ms); + }), + ]).finally(() => clearTimeout(timer)); +} + +async function runChromiumPage(chrome, pageURL, userDataDir) { + const child = spawn(chrome, [ + "--headless=new", + "--disable-gpu", + "--no-first-run", + "--no-default-browser-check", + `--user-data-dir=${userDataDir}`, + "--remote-debugging-port=0", + pageURL, + ], { stdio: ["ignore", "ignore", "pipe"] }); + const browserURL = await waitForDebuggerURL(child); + const ws = await connectWebSocket(await pageWebSocketURL(browserURL, pageURL)); + try { + await cdpCommand(ws, "Runtime.enable"); + for (let attempt = 0; attempt < 100; attempt += 1) { + const status = await evaluate(ws, "document.body && document.body.dataset.ok"); + if (status === "true") { + return JSON.parse(await evaluate(ws, "document.body.textContent")); + } + if (status === "false") { + const text = await evaluate(ws, "document.body.textContent"); + let payload; + try { + payload = JSON.parse(text); + } catch { + payload = { error: text }; + } + throw new Error(payload.error || text); + } + await delay(100); + } + throw new Error("Chromium benchmark timed out waiting for page result"); + } finally { + ws.close(); + await terminateProcess(child); + } +} + +async function measureChromium(bytes, bench, iterations, options) { + return measureBrowserEngine("chromium", bytes, bench, iterations, options); +} + +async function measureBrowserEngine(engineName, bytes, bench, iterations, options) { + const page = benchmarkPage(bench, iterations, engineName); + let postResultResolve; + const postResult = new Promise((resolvePromise) => { + postResultResolve = resolvePromise; + }); + const server = createServer(async (request, response) => { + try { + const url = new URL(request.url, "http://127.0.0.1"); + if (request.method === "POST" && url.pathname === "/result") { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => { + response.writeHead(200, { "content-type": "text/plain; charset=utf-8" }); + response.end("ok"); + postResultResolve(body); + }); + return; + } + if (url.pathname === "/" || url.pathname === "/index.html") { + response.writeHead(200, { "content-type": contentType(".html") }); + response.end(page); + return; + } + if (url.pathname === "/module.wasm") { + response.writeHead(200, { "content-type": contentType(".wasm") }); + response.end(bytes); + return; + } + if (url.pathname.startsWith("/support/wasm-gc/") && url.pathname.endsWith(".mjs")) { + const source = await readFile(join(repoRoot, url.pathname), "utf8"); + response.writeHead(200, { "content-type": contentType(url.pathname) }); + response.end(source); + return; + } + response.writeHead(404, { "content-type": "text/plain; charset=utf-8" }); + response.end("not found"); + } catch (err) { + response.writeHead(500, { "content-type": "text/plain; charset=utf-8" }); + response.end(err && err.stack ? err.stack : String(err)); + } + }); + const userDataDir = await mkdtemp(join(tmpdir(), "jerboa-wasm-gc-bench-chromium-")); + await new Promise((resolvePromise) => server.listen(0, "127.0.0.1", resolvePromise)); + const port = server.address().port; + let webkit = null; + try { + if (engineName === "webkit") { + webkit = spawnWebKitPage(`http://127.0.0.1:${port}/`); + const text = await withTimeout( + postResult, + 60000, + `WebKit benchmark timed out waiting for page result${webkit.output() ? `\n${webkit.output()}` : ""}` + ); + let body; + try { + body = JSON.parse(text); + } catch { + throw new Error(`WebKit benchmark returned invalid result payload: ${text}`); + } + if (!body.ok) { + throw new Error(body.payload && body.payload.error ? body.payload.error : text); + } + return body.payload; + } + const text = await runBrowserResultPage(engineName, `http://127.0.0.1:${port}/`, { + tmpPrefix: `jerboa-wasm-gc-bench-${engineName}-`, + }); + return JSON.parse(text); + } catch (err) { + if (err instanceof BrowserEngineUnavailable && options.allowMissingEngine) { + return { + engineName, + skipped: true, + skipReason: err.message, + }; + } + throw err; + } finally { + if (webkit) await terminateProcess(webkit.child); + server.close(); + if (typeof server.closeAllConnections === "function") { + server.closeAllConnections(); + } + await rm(userDataDir, { recursive: true, force: true }); + } +} + +async function measureEngine(engineName, bytes, bench, target, iterations, options) { + if (engineName === "node") { + return measureNode(bytes, bench, target, iterations); + } + if (engineName === "chromium") { + return measureChromium(bytes, bench, iterations, options); + } + if (engineName === "firefox" || engineName === "webkit" || engineName === "safari") { + return measureBrowserEngine(engineName, bytes, bench, iterations, options); + } + throw new Error(`unsupported benchmark engine ${engineName}`); +} + +function checkMaxBudget(failures, budgets, name, actual) { + const limit = budgets[name]; + if (limit === undefined || limit === null) return; + if (!(actual <= limit)) { + failures.push({ + name, + actual, + limit, + relation: "<=", + }); + } +} + +function budgetFailuresFor(result) { + const budgets = result.budgets ?? {}; + const failures = []; + checkMaxBudget(failures, budgets, "maxWasmBytes", result.wasmBytes); + checkMaxBudget(failures, budgets, "maxCompilerMs", result.compilerMs); + if (result.engine) { + checkMaxBudget(failures, budgets, "maxValidateNs", result.engine.validateNs); + checkMaxBudget(failures, budgets, "maxEngineCompileNs", result.engine.compileNs); + checkMaxBudget(failures, budgets, "maxInstantiateNs", result.engine.instantiateNs); + checkMaxBudget(failures, budgets, "maxRunTotalNs", result.engine.runTotalNs); + checkMaxBudget(failures, budgets, "maxRunMeanNs", result.engine.runMeanNs); + } + return failures; +} + +async function runCase(root, outDir, manifest, bench, target, optLevel, engineName, options) { + const base = `${bench.id}-${target}-O${optLevel}`; + const wasmPath = resolve(outDir, `${base}.wasm`); + const sizePath = resolve(outDir, `${base}.size.sexp`); + const buildArgs = [ + "wasm", "build", + "--backend", "gc", + "--target", target, + `-O${optLevel}`, + "--emit-size-report", sizePath, + resolve(root, bench.source), + "-o", wasmPath, + ]; + const compileStart = performance.now(); + await runCommand("./bin/jerboa", buildArgs, { cwd: root }); + const compilerMs = performance.now() - compileStart; + + const bytes = await readFile(wasmPath); + const iterations = options.quick ? 1 : (bench.iterations ?? manifest.defaultIterations ?? 1); + const measured = await measureEngine(engineName, bytes, bench, target, iterations, options); + if (!measured.skipped && bench.expected !== undefined && measured.result !== String(bench.expected)) { + throw new Error(`${bench.id}: ${engineName} got ${measured.result}, expected ${bench.expected}`); + } + + let sizeReportText = ""; + try { + sizeReportText = await readFile(sizePath, "utf8"); + } catch { + sizeReportText = ""; + } + + const result = { + id: bench.id, + source: bench.source, + export: bench.export, + engineName: measured.engineName, + skipped: measured.skipped ?? false, + skipReason: measured.skipReason ?? null, + target: measured.target ?? target, + requestedTarget: target, + optLevel, + iterations, + wasmBytes: bytes.length, + compilerMs, + engine: measured.metrics ?? null, + expected: bench.expected ?? null, + result: measured.result ?? null, + artifacts: { + wasm: wasmPath, + sizeReport: sizePath, + }, + sizeReportText, + budgets: bench.budgets ?? {}, + }; + result.budgetFailures = measured.skipped ? [] : budgetFailuresFor(result); + return result; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const [manifestPathText, reportPathText] = options.positionals; + const root = process.cwd(); + const manifestPath = resolve(root, manifestPathText); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + validateManifest(manifest); + if (options.checkManifest) { + console.log("ok wasm-gc benchmark manifest"); + return; + } + + const reportPath = reportPathText ? resolve(root, reportPathText) : null; + const outDir = resolve(root, ".tmp/wasm-gc-bench"); + await mkdir(outDir, { recursive: true }); + + const results = []; + for (const bench of manifest.cases) { + for (const target of bench.targets) { + for (const optLevel of bench.optLevels) { + for (const engineName of options.engines) { + results.push(await runCase(root, outDir, manifest, bench, target, optLevel, engineName, options)); + } + } + } + } + + const report = { + schema: REPORT_SCHEMA, + manifest: manifestPathText, + quick: options.quick, + engines: options.engines, + runtime: { + node: process.version, + v8: process.versions.v8, + platform: process.platform, + arch: process.arch, + }, + results, + }; + const text = `${JSON.stringify(report, null, 2)}\n`; + if (reportPath) { + await mkdir(dirname(reportPath), { recursive: true }); + await writeFile(reportPath, text); + } else { + process.stdout.write(text); + } + const budgetFailures = results.flatMap((result) => + (result.budgetFailures ?? []).map((failure) => ({ + id: result.id, + engineName: result.engineName, + target: result.requestedTarget, + optLevel: result.optLevel, + ...failure, + })) + ); + if (budgetFailures.length > 0) { + throw new Error(`benchmark budgets exceeded: ${JSON.stringify(budgetFailures)}`); + } +} + +main().catch((err) => { + console.error(err && err.stack ? err.stack : String(err)); + process.exit(1); +}); new file mode 100644 --- /dev/null +++ b/support/wasm-gc/browser-engine.mjs @@ -0,0 +1,364 @@ +import { createServer } from "node:http"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn, spawnSync } from "node:child_process"; + +export const SUPPORTED_BROWSER_ENGINES = Object.freeze(["chromium", "firefox", "webkit", "safari"]); + +export class BrowserEngineUnavailable extends Error { + constructor(engineName, message) { + super(message); + this.name = "BrowserEngineUnavailable"; + this.engineName = engineName; + } +} + +function commandPath(command) { + const probe = spawnSync("which", [command], { encoding: "utf8" }); + return probe.status === 0 ? probe.stdout.trim() : null; +} + +function chromiumPath() { + if (process.env.CHROME_PATH) return process.env.CHROME_PATH; + const absoluteCandidates = [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ]; + for (const candidate of absoluteCandidates) { + if (existsSync(candidate)) return candidate; + } + for (const candidate of [ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + ]) { + const path = commandPath(candidate); + if (path) return path; + } + return null; +} + +function firefoxDriverPath() { + if (process.env.GECKODRIVER_PATH) return process.env.GECKODRIVER_PATH; + return commandPath("geckodriver"); +} + +function firefoxBinaryPath() { + if (process.env.FIREFOX_PATH) return process.env.FIREFOX_PATH; + for (const candidate of [ + "/Applications/Firefox.app/Contents/MacOS/firefox", + "/Applications/Firefox Developer Edition.app/Contents/MacOS/firefox", + ]) { + if (existsSync(candidate)) return candidate; + } + return commandPath("firefox"); +} + +function safariDriverPath() { + if (process.env.SAFARIDRIVER_PATH) return process.env.SAFARIDRIVER_PATH; + for (const candidate of [ + "/System/Cryptexes/App/usr/bin/safaridriver", + "/usr/bin/safaridriver", + ]) { + if (existsSync(candidate)) return candidate; + } + return commandPath("safaridriver"); +} + +function delay(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +async function freePort() { + const server = createServer(); + await new Promise((resolvePromise, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolvePromise); + }); + const port = server.address().port; + await new Promise((resolvePromise) => server.close(resolvePromise)); + return port; +} + +function terminateProcess(child) { + return new Promise((resolvePromise) => { + if (!child || child.exitCode !== null || child.signalCode !== null) { + resolvePromise(); + return; + } + child.once("close", () => resolvePromise()); + child.kill(); + setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, 3000).unref(); + }); +} + +function waitForDebuggerURL(child, engineLabel) { + return new Promise((resolvePromise, reject) => { + let done = false; + let stderr = ""; + const timer = setTimeout(() => { + if (done) return; + done = true; + reject(new BrowserEngineUnavailable("chromium", stderr || `${engineLabel} did not expose a DevTools endpoint`)); + }, 10000); + child.stderr.on("data", (chunk) => { + if (done) return; + stderr += String(chunk); + const match = stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/); + if (match) { + done = true; + clearTimeout(timer); + resolvePromise(match[1]); + } + }); + child.on("error", (err) => { + if (done) return; + done = true; + clearTimeout(timer); + reject(new BrowserEngineUnavailable("chromium", err.message)); + }); + child.on("exit", (code) => { + if (done) return; + done = true; + clearTimeout(timer); + reject(new BrowserEngineUnavailable("chromium", stderr || `${engineLabel} exited before DevTools was ready: ${code}`)); + }); + }); +} + +async function pageWebSocketURL(browserURL, pageURL) { + const endpoint = new URL(browserURL); + const listURL = `http://${endpoint.host}/json/list`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const response = await fetch(listURL).catch(() => null); + if (response?.ok) { + const targets = await response.json(); + const page = targets.find((target) => + target.type === "page" && target.url === pageURL && target.webSocketDebuggerUrl + ); + if (page) return page.webSocketDebuggerUrl; + } + await delay(100); + } + throw new Error("Chromium page target was not available through DevTools"); +} + +function connectWebSocket(url) { + return new Promise((resolvePromise, reject) => { + const ws = new WebSocket(url); + ws.addEventListener("open", () => resolvePromise(ws), { once: true }); + ws.addEventListener("error", reject, { once: true }); + }); +} + +function cdpCommand(ws, method, params = {}) { + const id = cdpCommand.nextId++; + ws.send(JSON.stringify({ id, method, params }));