Skip to content

Commit 3666d6a

Browse files
panvaaduh95
authored andcommitted
test: schedule WPT variants individually
Discover WPT tasks through their existing JavaScript drivers so the Python runner can schedule, report, and rerun generated paths. Assisted-by: Codex Signed-off-by: Filip Skokan <panva.ip@gmail.com> PR-URL: #65984 Fixes: #51854 Reviewed-By: Aviv Keller <me@aviv.sh> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
1 parent a05023f commit 3666d6a

11 files changed

Lines changed: 531 additions & 36 deletions

File tree

test/common/wpt.js

Lines changed: 73 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,8 @@ class ReportResult {
9696
// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
9797
// for more details.
9898
class WPTReport {
99-
constructor(testPath) {
100-
this.filename = `report-${testPath.replaceAll('/', '-')}.json`;
99+
constructor(testPath, suffix = '') {
100+
this.filename = `report-${testPath.replaceAll('/', '-')}${suffix}.json`;
101101
this.filepath = path.join(__dirname, `../../out/wpt/${this.filename}`);
102102
/** @type {Map<string, ReportResult>} */
103103
this.results = new Map();
@@ -461,12 +461,14 @@ class WPTTestSpec {
461461
/**
462462
* Whether a command line argument selects this spec. Accepts the source file
463463
* name, which selects every global and variant generated from it, or a test
464-
* path as printed alongside the results, which selects only this one.
464+
* path as printed alongside the results. Omitting its query selects all
465+
* variants of that global.
465466
* @param {string} arg
466467
* @returns {boolean}
467468
*/
468469
isSelectedBy(arg) {
469-
if (arg === this.getTestPath()) {
470+
const testPath = this.getTestPath();
471+
if (arg === testPath || arg === testPath.split('?')[0]) {
470472
return true;
471473
}
472474
const [filename, variant = ''] = arg.split('?');
@@ -608,7 +610,7 @@ class StatusLoader {
608610
return result;
609611
}
610612

611-
load() {
613+
load(source) {
612614
const dir = path.join(__dirname, '..', 'wpt');
613615
let result;
614616

@@ -625,7 +627,7 @@ class StatusLoader {
625627
this.rules.addRules(result);
626628

627629
const subDir = fixtures.path('wpt', this.path);
628-
const list = this.grep(subDir);
630+
const list = source === undefined ? this.grep(subDir) : [path.join(subDir, source)];
629631
for (const file of list) {
630632
const relativePath = path.relative(subDir, file);
631633
const match = this.rules.match(relativePath);
@@ -813,14 +815,29 @@ const backends = {
813815
};
814816

815817
class WPTRunner {
816-
constructor(path, {
817-
concurrency = os.availableParallelism() - 1 || 1,
818-
backend = 'thread',
819-
} = {}) {
818+
constructor(path, options = {}) {
819+
let {
820+
concurrency = os.availableParallelism() - 1 || 1,
821+
backend = 'thread',
822+
} = options;
820823
if (!Number.isInteger(concurrency) || concurrency < 1) {
821824
throw new TypeError('WPT concurrency must be a positive integer');
822825
}
823826

827+
if (process.env.NODE_TEST_WPT !== undefined) {
828+
this.managed = JSON.parse(process.env.NODE_TEST_WPT);
829+
if (!this.managed || !['list', 'run'].includes(this.managed.mode) ||
830+
(this.managed.mode === 'run' &&
831+
(['source', 'key'].some((key) =>
832+
typeof this.managed[key] !== 'string' || !this.managed[key]) ||
833+
(this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) {
834+
throw new Error('Invalid WPT runner configuration');
835+
}
836+
}
837+
this.isListing = this.managed?.mode === 'list';
838+
this.serial = options.concurrency === 1;
839+
if (this.managed?.mode === 'run') concurrency = 1;
840+
824841
// RISC-V has very limited virtual address space in the currently common
825842
// sv39 mode, in which we can only create a very limited number of wasm
826843
// memories(27 from a fresh node repl). Limit the concurrency to avoid
@@ -860,7 +877,7 @@ class WPTRunner {
860877
this.initScript = null;
861878

862879
this.status = new StatusLoader(path);
863-
this.status.load();
880+
this.status.load(this.managed?.mode === 'run' ? this.managed.source : undefined);
864881
this.statusFile = this.status.statusFile;
865882
this.specs = new Set(this.status.specs);
866883

@@ -872,8 +889,9 @@ class WPTRunner {
872889

873890
this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 };
874891

875-
if (process.env.WPT_REPORT != null) {
876-
this.report = new WPTReport(path);
892+
if (process.env.WPT_REPORT != null && !this.isListing) {
893+
const suffix = this.managed ? `-${process.env.TEST_SERIAL_ID || process.pid}` : '';
894+
this.report = new WPTReport(path, suffix);
877895
}
878896
}
879897

@@ -958,7 +976,40 @@ class WPTRunner {
958976
// TODO(joyeecheung): work with the upstream to port more tests in .html
959977
// to .js.
960978
async runJsTests() {
979+
if (this.isListing) {
980+
const groups = new Map();
981+
for (const spec of this.specs) {
982+
const key = spec.getStatusKey();
983+
if (!groups.has(key)) groups.set(key, []);
984+
groups.get(key).push(spec);
985+
}
986+
const tests = [...groups.values()].flatMap((specs) => {
987+
// Strict expected failures are checked across all query variants.
988+
// Keep that group together; all other variants are independent tasks.
989+
const grouped = specs.some((spec) =>
990+
spec.failedTests.some((name) => isUnexpectedPass(spec, name)));
991+
return (grouped ? [specs[0]] : specs).map((spec) => {
992+
const selector = grouped ? spec.getTestPath().split('?')[0] : spec.getTestPath();
993+
return {
994+
source: spec.filename.split(path.sep).join('/'),
995+
key: spec.getStatusKey(),
996+
...(grouped ? {} : { variant: spec.variant }),
997+
id: selector.slice(this.path.length + 1),
998+
selector,
999+
};
1000+
});
1001+
});
1002+
console.log(`NODE_TEST_WPT_MANIFEST:${JSON.stringify({
1003+
version: 1,
1004+
serial: this.serial,
1005+
tests: tests.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)),
1006+
})}`);
1007+
return;
1008+
}
9611009
const queue = this.buildQueue();
1010+
if (this.managed && queue.length === 0) {
1011+
console.log('1..0 # SKIP No runnable WPT variants');
1012+
}
9621013

9631014
const run = limit(this.concurrency);
9641015
const jobs = [];
@@ -1313,11 +1364,19 @@ class WPTRunner {
13131364
buildQueue() {
13141365
const queue = [];
13151366
this.skippedSpecCount = 0;
1316-
const arg = process.argv[2];
1367+
const key = this.managed?.mode === 'run' ? this.managed.key : undefined;
1368+
const variant = this.managed?.variant;
1369+
const matches = (spec) => spec.getStatusKey() === key &&
1370+
(variant === undefined || spec.variant === variant);
1371+
const arg = key === undefined ? process.argv[2] : undefined;
1372+
if (key !== undefined && ![...this.specs].some(matches)) {
1373+
throw new Error(`${key}${variant ?? ''} not found!`);
1374+
}
13171375
if (this.inspectBrk && !arg) {
13181376
throw new Error('WPT_INSPECT requires a WPT test path');
13191377
}
13201378
for (const spec of this.specs) {
1379+
if (key !== undefined && !matches(spec)) continue;
13211380
if (arg) {
13221381
if (spec.isSelectedBy(arg)) {
13231382
queue.push(spec);
Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
'use strict';
2+
3+
const common = require('../common');
4+
const assert = require('assert');
5+
const fs = require('fs');
6+
const path = require('path');
7+
const { spawnSync } = require('child_process');
8+
const tmpdir = require('../common/tmpdir');
9+
10+
if (process.env.NODE_TEST_WPT_REPORT_DIR) {
11+
const { WPTRunner } = require('../common/wpt');
12+
const runner = new WPTRunner('html/webappapis/atob');
13+
runner.report.filepath = path.join(process.env.NODE_TEST_WPT_REPORT_DIR, runner.report.filename);
14+
runner.runJsTests();
15+
} else if (process.env.NODE_TEST_WPT_QUERY_PROBE) {
16+
const { WPTRunner, WPTTestSpec } = require('../common/wpt');
17+
const runner = new WPTRunner('compression');
18+
if (runner.managed?.mode === 'run') assert.strictEqual(runner.concurrency, 1);
19+
runner.specs = new Set(['?pass', '?fail'].map((query) => {
20+
const spec = new WPTTestSpec('compression', 'compression-bad-chunks.any.js', [], query, 'window');
21+
spec.failedTests = ['expected across queries'];
22+
if (process.env.NODE_TEST_WPT_QUERY_PROBE === 'flaky' ||
23+
(process.env.NODE_TEST_WPT_QUERY_PROBE === 'mixed' && query === '?pass')) {
24+
spec.flakyTests = [...spec.failedTests];
25+
}
26+
return spec;
27+
}));
28+
runner.setScriptModifier((script) => {
29+
if (!script.filename.endsWith('compression-bad-chunks.any.js')) return;
30+
script.code = `test(() => assert_true(${process.env.NODE_TEST_WPT_QUERY_PROBE === 'missing'} ||
31+
location.search === '?pass'), 'expected across queries');`;
32+
});
33+
runner.runJsTests();
34+
} else {
35+
main();
36+
}
37+
38+
function main() {
39+
tmpdir.refresh();
40+
const env = { ...process.env };
41+
for (const key of ['NODE_TEST_WPT', 'WPT_REPORT', 'WPT_INSPECT']) delete env[key];
42+
const driver = (name) => path.join(__dirname, '../wpt', `test-${name}.js`);
43+
function invoke(file, config, overrides = {}, status = 0) {
44+
const result = spawnSync(process.execPath, [file], {
45+
env: { ...env, ...overrides, NODE_TEST_WPT: JSON.stringify(config) },
46+
encoding: 'utf8', timeout: common.platformTimeout(10_000),
47+
maxBuffer: 10 * 1024 * 1024,
48+
});
49+
assert.ifError(result.error);
50+
assert.strictEqual(result.status, status, result.stdout + result.stderr);
51+
return result.stdout + result.stderr;
52+
}
53+
54+
function discover(name, overrides, file = driver(name)) {
55+
const stdout = invoke(file, { mode: 'list' }, overrides);
56+
const lines = stdout.split('\n').filter((line) => line.startsWith('NODE_TEST_WPT_MANIFEST:'));
57+
assert.strictEqual(lines.length, 1);
58+
assert.doesNotMatch(stdout, /\[PASS\]/);
59+
const manifest = JSON.parse(lines[0].slice('NODE_TEST_WPT_MANIFEST:'.length));
60+
assert.strictEqual(manifest.version, 1);
61+
assert.strictEqual(new Set(manifest.tests.map((test) => test.id)).size, manifest.tests.length);
62+
return manifest;
63+
}
64+
65+
const atob = discover('atob');
66+
assert.strictEqual(atob.serial, false);
67+
assert.deepStrictEqual(atob.tests.map((test) => test.id), ['base64.any.html', 'base64.any.worker.html']);
68+
const reportRoot = path.join(tmpdir.path, 'reports');
69+
const canReport = ['darwin', 'linux', 'win32'].includes(process.platform);
70+
if (canReport) fs.mkdirSync(reportRoot, { recursive: true });
71+
for (const [index, group] of atob.tests.entries()) {
72+
const serial = `group-probe-${process.pid}-${index}`;
73+
const reportPath = path.join(reportRoot, `report-html-webappapis-atob-${serial}.json`);
74+
try {
75+
const stdout = invoke(canReport ? __filename : driver('atob'),
76+
{ mode: 'run', source: group.source, key: group.key, variant: group.variant }, canReport ? {
77+
WPT_REPORT: '1', TEST_SERIAL_ID: serial, NODE_TEST_WPT_REPORT_DIR: reportRoot,
78+
} : {});
79+
const results = stdout.split('\n').filter((line) => line.startsWith('[PASS]'));
80+
assert.ok(results.length > 0);
81+
assert.ok(results.every((line) => line.includes(`${group.id}:`)));
82+
if (canReport) {
83+
const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
84+
assert.deepStrictEqual(report.results.map((result) => result.test),
85+
[`/html/webappapis/atob/${group.id}`]);
86+
}
87+
} finally {
88+
fs.rmSync(reportPath, { force: true });
89+
}
90+
}
91+
92+
const encoding = discover('encoding');
93+
const queryGroups = encoding.tests.filter((test) => test.source === 'api-invalid-label.any.js');
94+
assert.strictEqual(queryGroups.length, 8);
95+
assert.deepStrictEqual([...new Set(queryGroups.map((test) => test.key))],
96+
['api-invalid-label.any.html', 'api-invalid-label.any.worker.html']);
97+
const timers = discover('timers');
98+
assert.strictEqual(timers.serial, true);
99+
const skipped = timers.tests.find((test) => test.source === 'negative-settimeout.any.js');
100+
assert.ok(skipped);
101+
const skippedOutput = invoke(driver('timers'), { mode: 'run', source: skipped.source, key: skipped.key });
102+
assert.match(skippedOutput, /\[SKIPPED\].*unreliable in Node\.js/);
103+
assert.match(skippedOutput, /1\.\.0 # SKIP/);
104+
assert.doesNotMatch(skippedOutput, /\[PASS\]/);
105+
106+
if (common.hasSQLite) {
107+
const root = path.join(tmpdir.path, 'discovery');
108+
const directory = path.join(root, '.tmp.probe');
109+
fs.mkdirSync(directory, { recursive: true });
110+
const sentinel = path.join(directory, 'sentinel');
111+
fs.writeFileSync(sentinel, 'preserved');
112+
assert.strictEqual(discover('webstorage', { NODE_TEST_DIR: root, TEST_SERIAL_ID: 'probe' }).serial, true);
113+
assert.strictEqual(fs.readFileSync(sentinel, 'utf8'), 'preserved');
114+
}
115+
116+
const config = { mode: 'run', source: 'compression-bad-chunks.any.js', key: 'compression-bad-chunks.any.html' };
117+
for (const probe of ['combined', 'mixed']) {
118+
const strict = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: probe }, __filename);
119+
assert.deepStrictEqual(strict.tests, [{
120+
source: config.source, key: config.key, id: config.key, selector: `compression/${config.key}`,
121+
}]);
122+
}
123+
const flaky = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }, __filename);
124+
assert.deepStrictEqual(flaky.tests.map((test) => test.variant).sort(), ['?fail', '?pass']);
125+
assert.deepStrictEqual(flaky.tests.map((test) => test.id).sort(),
126+
[`${config.key}?fail`, `${config.key}?pass`]);
127+
const single = invoke(__filename, { ...config, variant: '?pass' }, { NODE_TEST_WPT_QUERY_PROBE: 'flaky' });
128+
assert.match(single, /\.any\.html\?pass:/);
129+
assert.doesNotMatch(single, /\.any\.html\?fail:/);
130+
const combined = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'combined' });
131+
assert.match(combined, /\.any\.html\?pass:/);
132+
assert.match(combined, /\.any\.html\?fail:/);
133+
const missing = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'missing' }, 1);
134+
assert.match(missing, /Found 2 unexpected passes/);
135+
136+
for (const query of ['', '?fail']) {
137+
const direct = spawnSync(process.execPath, [__filename, `compression/${config.key}${query}`], {
138+
env: { ...env, NODE_TEST_WPT_QUERY_PROBE: 'combined' },
139+
encoding: 'utf8', timeout: common.platformTimeout(10_000),
140+
});
141+
assert.ifError(direct.error);
142+
assert.strictEqual(direct.status, 0, direct.stdout + direct.stderr);
143+
assert.match(direct.stdout, /\.any\.html\?fail:/);
144+
if (query) assert.doesNotMatch(direct.stdout, /\.any\.html\?pass:/);
145+
else assert.match(direct.stdout, /\.any\.html\?pass:/);
146+
}
147+
}

0 commit comments

Comments
 (0)