Skip to content

Commit b34d681

Browse files
test: add browser test harness (#563)
This commit adds a browser test harness to run the tests in the browser. ## Motivation We are heavily using wasi-libc on browsers but we don't have any test for the case in wasi-libc. In theory, there should be no behavioral difference between wasmtime and browsers (as long as WASI implementations are correct) but browsers have their own limitations in their Wasm engines. For example, memory.atomic.wait32 is not permitted on the main thread. We are working on adding some mitigation for such browser-specific issues (#562) and this test harness will help to validate the fix.
1 parent abe6062 commit b34d681

File tree

8 files changed

+536
-0
lines changed

8 files changed

+536
-0
lines changed

.github/workflows/main.yml

+5
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,11 @@ jobs:
142142
TARGET_TRIPLE=wasm32-wasip2 make test
143143
TARGET_TRIPLE=wasm32-wasi-threads make test
144144
TARGET_TRIPLE=wasm32-wasip1-threads make test
145+
146+
npm -C scripts/browser-test install
147+
npx -C scripts/browser-test playwright install chromium-headless-shell
148+
ENGINE="$PWD/scripts/browser-test/harness.mjs" TARGET_TRIPLE=wasm32-wasip1 make test
149+
ENGINE="$PWD/scripts/browser-test/harness.mjs" TARGET_TRIPLE=wasm32-wasip1-threads make test
145150
# The older version of Clang does not provide the expected symbol for the
146151
# test entrypoints: `undefined symbol: __main_argc_argv`.
147152
# The older (<15.0.7) version of wasm-ld does not provide `__heap_end`,

test/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
build
22
run
3+
node_modules

test/README.md

+10
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,16 @@ fs # a directory containing any test-created files
3535
output.log # the captured printed output--only for errors
3636
```
3737

38+
### Running tests in the browser
39+
40+
To run a test in the browser, use the `scripts/browser-test/harness.mjs` as `ENGINE`
41+
42+
```sh
43+
$ npm -C scripts/browser-test install
44+
$ npx -C scripts/browser-test playwright install chromium-headless-shell
45+
$ make ENGINE="$PWD/scripts/browser-test/harness.mjs" TARGET_TRIPLE=...
46+
```
47+
3848
### Adding tests
3949

4050
To add a test, create a new C file in [`src/misc`]:

test/scripts/browser-test/harness.mjs

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/usr/bin/env node
2+
3+
/*
4+
* Run a WASI-compatible test program in the browser.
5+
*
6+
* This script behaves like `wasmtime` but runs given WASI-compatible test
7+
* program in the browser.
8+
*
9+
* Example:
10+
* $ ./harness.mjs check.wasm
11+
*/
12+
13+
import { parseArgs } from 'node:util';
14+
import { createServer } from 'node:http';
15+
import { fileURLToPath } from 'node:url';
16+
import { dirname, join } from 'node:path';
17+
import { readFileSync } from 'node:fs';
18+
import url from "node:url";
19+
import { chromium } from 'playwright';
20+
21+
const SKIP_TESTS = [
22+
// "poll_oneoff" can't be implemented in the browser
23+
"libc-test/functional/pthread_cond",
24+
// atomic.wait32 can't be executed on the main thread
25+
"libc-test/functional/pthread_mutex",
26+
"libc-test/functional/pthread_tsd",
27+
// XFAIL: @bjorn3/browser_wasi_shim doesn't support symlinks for now
28+
"misc/fts",
29+
];
30+
31+
/**
32+
* @param {{wasmPath: string, port: number}}
33+
* @returns {Promise<{server: import('node:http').Server, port: number}>}
34+
*/
35+
async function startServer({ wasmPath, port }) {
36+
const server = createServer((req, res) => {
37+
// Set required headers for SharedArrayBuffer
38+
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
39+
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
40+
41+
let filePath;
42+
const parsed = url.parse(req.url, true);
43+
const pathname = parsed.pathname;
44+
if (pathname === "/target.wasm") {
45+
// Serve the test target Wasm file
46+
filePath = wasmPath;
47+
res.setHeader('Content-Type', 'application/wasm');
48+
} else {
49+
// Serve other resources
50+
const __dirname = dirname(fileURLToPath(import.meta.url));
51+
filePath = join(__dirname, pathname);
52+
const contentTypes = {
53+
"mjs": "text/javascript",
54+
"js": "text/javascript",
55+
"html": "text/html",
56+
}
57+
res.setHeader('Content-Type', contentTypes[pathname.split('.').pop()] || 'text/plain');
58+
}
59+
60+
try {
61+
const content = readFileSync(filePath);
62+
res.end(content);
63+
} catch (error) {
64+
res.statusCode = 404;
65+
res.end('Not found');
66+
}
67+
});
68+
69+
return new Promise((resolve) => {
70+
server.listen(port, () => {
71+
const port = server.address().port;
72+
resolve({ server, port });
73+
});
74+
});
75+
}
76+
77+
/** @param {number} port */
78+
function buildUrl(port) {
79+
return `http://localhost:${port}/run-test.html`;
80+
}
81+
82+
/** @param {import('playwright').Page} page */
83+
/** @param {number} port */
84+
/** @returns {Promise<{passed: boolean, error?: string}>} */
85+
async function runTest(page, port) {
86+
const url = buildUrl(port);
87+
const onExit = new Promise((resolve) => {
88+
page.exposeFunction("exitTest", resolve);
89+
});
90+
await page.goto(url);
91+
return onExit;
92+
}
93+
94+
async function main() {
95+
// Parse and interpret a subset of the wasmtime CLI options used by the tests
96+
const args = parseArgs({
97+
args: process.argv.slice(2),
98+
allowPositionals: true,
99+
options: {
100+
// MARK: wasmtime CLI options
101+
wasi: {
102+
type: "string",
103+
multiple: true,
104+
},
105+
dir: {
106+
type: "string",
107+
multiple: true,
108+
},
109+
// MARK: For debugging purposes
110+
headful: {
111+
type: "boolean",
112+
default: false,
113+
},
114+
port: {
115+
type: "string",
116+
default: "0",
117+
}
118+
}
119+
});
120+
121+
const wasmPath = args.positionals[0];
122+
if (!wasmPath) {
123+
console.error('Test path not specified');
124+
return 1;
125+
}
126+
127+
if (SKIP_TESTS.some(test => wasmPath.includes(test + "."))) {
128+
// Silently skip tests that are known to fail in the browser
129+
return 0;
130+
}
131+
132+
if (args.values.dir && args.values.dir.length > 0) {
133+
// Silently skip tests that require preopened directories for now
134+
// as it adds more complexity to the harness and file system emulation
135+
// is not our primary testing target.
136+
return 0;
137+
}
138+
139+
// Start a HTTP server to serve the test files
140+
const { server, port } = await startServer({ wasmPath, port: parseInt(args.values.port) });
141+
142+
const browser = await chromium.launch();
143+
const page = await browser.newPage();
144+
145+
try {
146+
if (args.values.headful) {
147+
// Run in headful mode to allow manual testing
148+
console.log(`Please visit ${buildUrl(port)}`);
149+
console.log('Press Ctrl+C to stop');
150+
await new Promise(resolve => process.on('SIGINT', resolve));
151+
return 0;
152+
}
153+
154+
// Run in headless mode
155+
const result = await runTest(page, port);
156+
if (!result.passed) {
157+
console.error('Test failed:', result.error);
158+
console.error(`Hint: You can debug the test by running it in headful mode by passing --headful
159+
$ ${process.argv.join(' ')} --headful`);
160+
return 1;
161+
}
162+
return 0;
163+
} catch (error) {
164+
console.error('Test failed:', error);
165+
return 1;
166+
} finally {
167+
await browser.close();
168+
server.close();
169+
}
170+
}
171+
172+
process.exit(await main());

test/scripts/browser-test/package-lock.json

+75
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"dependencies": {
3+
"playwright": "^1.49.1"
4+
}
5+
}
+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<!DOCTYPE html>
2+
<html>
3+
4+
<head>
5+
<title>wasi-libc Browser Tests</title>
6+
</head>
7+
8+
<body>
9+
<h1>wasi-libc Browser Tests</h1>
10+
<div id="results"></div>
11+
<script type="module">
12+
import { runWasmTest } from "./run-test.mjs";
13+
function exitTest(result) {
14+
if (typeof window.exitTest === 'function') {
15+
window.exitTest(result);
16+
}
17+
}
18+
async function runTests() {
19+
const resultsDiv = document.getElementById('results');
20+
21+
try {
22+
const passed = await runWasmTest("target.wasm");
23+
resultsDiv.innerHTML = passed ?
24+
'<p style="color: green">Test passed</p>' :
25+
'<p style="color: red">Test failed</p>';
26+
exitTest({ passed });
27+
} catch (error) {
28+
console.error(error);
29+
resultsDiv.innerHTML = `<p style="color: red">Error: ${error.message}</p>`;
30+
exitTest({ passed: false, error: error.message });
31+
}
32+
}
33+
34+
runTests();
35+
</script>
36+
</body>
37+
38+
</html>

0 commit comments

Comments
 (0)