forked from chromium/chromium
-
Notifications
You must be signed in to change notification settings - Fork 0
/
testharnessreport.js
352 lines (315 loc) · 13.3 KB
/
testharnessreport.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
/*
* This file is intended for vendors to implement code needed to integrate
* testharness.js tests with their own test systems.
*
* Typically such integration will attach callbacks when each test is
* has run, using add_result_callback(callback(test)), or when the whole test
* file has completed, using add_completion_callback(callback(tests,
* harness_status)).
*
* For more documentation about the callback functions and the
* parameters they are called with, see testharness.js, or the docs at:
* https://web-platform-tests.org/writing-tests/testharness-api.html
*/
(function() {
let outputDocument = document;
let didDispatchLoadEvent = false;
let localPathRegExp = undefined;
// Setup for Blink JavaScript tests. self.testRunner is expected to be
// present when tests are run.
if (self.testRunner) {
testRunner.dumpAsText();
testRunner.waitUntilDone();
testRunner.setPopupBlockingEnabled(false);
testRunner.setDumpJavaScriptDialogs(false);
// Some tests intentionally load mixed content in order to test the
// referrer policy, so WebKitAllowRunningInsecureContent must be set
// to true or else the load would be blocked.
const paths = [
'service-workers/service-worker/fetch-event-referrer-policy.https.html',
'beacon/headers/header-referrer-no-referrer-when-downgrade.https.html',
'beacon/headers/header-referrer-strict-origin-when-cross-origin.https.html',
'beacon/headers/header-referrer-strict-origin.https.html',
'beacon/headers/header-referrer-unsafe-url.https.html',
];
for (const path of paths) {
if (document.URL.endsWith(path)) {
testRunner.overridePreference('WebKitAllowRunningInsecureContent', true);
break;
}
}
}
if (document.URL.startsWith('file:///')) {
const index = document.URL.indexOf('/external/wpt');
if (index >= 0) {
const localPath = document.URL.substring(
'file:///'.length, index + '/external/wpt'.length);
localPathRegExp = new RegExp(localPath.replace(/(\W)/g, '\\$1'), 'g');
}
}
window.addEventListener('load', loadCallback, {'once': true});
setup({
// The default output formats test results into an HTML table, but for
// the Blink layout test runner, we dump the results as text in the
// completion callback, so we disable the default output.
'output': false,
// The Blink layout test runner has its own timeout mechanism.
'explicit_timeout': true
});
add_start_callback(startCallback);
add_completion_callback(completionCallback);
/** Loads an automation script if necessary. */
function loadCallback() {
didDispatchLoadEvent = true;
if (isWPTManualTest()) {
setTimeout(loadAutomationScript, 0);
}
}
/** Checks whether the current path is a manual test in WPT. */
function isWPTManualTest() {
// Here we assume that if wptserve is running, then the hostname
// is web-platform.test.
const path = location.pathname;
if (location.hostname == 'web-platform.test' &&
/.*-manual(\.sub)?(\.https)?(\.tentative)?\.html$/.test(path)) {
return true;
}
// If the file is loaded locally via file://, it must include
// the wpt directory in the path.
return /\/external\/wpt\/.*-manual(\.sub)?(\.https)?(\.tentative)?\.html$/.test(path);
}
/** Loads the WPT automation script for the current test, if applicable. */
function loadAutomationScript() {
const pathAndBase = pathAndBaseNameInWPT();
if (!pathAndBase) {
return;
}
let automationPath = location.pathname.replace(
/\/external\/wpt\/.*$/, '/external/wpt_automation');
if (location.hostname == 'web-platform.test') {
automationPath = '/wpt_automation';
}
// Export importAutomationScript for use by the automation scripts.
window.importAutomationScript = function(relativePath) {
const script = document.createElement('script');
script.src = automationPath + relativePath;
document.head.appendChild(script);
};
let src;
if (pathAndBase.startsWith('/fullscreen/')) {
// Fullscreen tests all use the same automation script.
src = automationPath + '/fullscreen/auto-click.js';
} else if (pathAndBase.startsWith('/file-system-access/local_')) {
// local_ File System Access tests all use the same automation script.
src = automationPath + '/file-system-access/auto-pick-folder.js';
} else if (pathAndBase.startsWith('/file-system-access/')) {
// Per-test automation scripts.
src = automationPath + pathAndBase + '-automation.sub.js';
} else if (
pathAndBase.startsWith('/css/') ||
pathAndBase.startsWith('/pointerevents/') ||
pathAndBase.startsWith('/uievents/') ||
pathAndBase.startsWith('/html/') ||
pathAndBase.startsWith('/input-events/') ||
pathAndBase.startsWith('/css/selectors/') ||
pathAndBase.startsWith('/css/cssom-view/') ||
pathAndBase.startsWith('/css/css-scroll-snap/') ||
pathAndBase.startsWith('/dom/events/') ||
pathAndBase.startsWith('/feature-policy/experimental-features/') ||
pathAndBase.startsWith('/permissions-policy/experimental-features/')) {
// Per-test automation scripts.
src = automationPath + pathAndBase + '-automation.js';
} else {
return;
}
const script = document.createElement('script');
script.src = src;
document.head.appendChild(script);
}
/**
* Sets the output document based on the given properties.
* Usually the output document is the current document, but it could be
* a separate document in some cases.
*/
function startCallback(properties) {
if (properties.output_document) {
outputDocument = properties.output_document;
}
}
/**
* Adds results to the page in a manner that allows dumpAsText to produce
* readable test results.
*/
function completionCallback(tests, harness_status) {
const xhtmlNS = 'http://www.w3.org/1999/xhtml';
// Create element to hold results.
const resultsElement = outputDocument.createElementNS(xhtmlNS, 'pre');
resultsElement.style.whiteSpace = 'pre-wrap';
resultsElement.style.lineHeight = '1.5';
// Declare result string.
let resultStr = 'This is a testharness.js-based test.\n';
// Check harness_status. If it is not 0, tests did not execute
// correctly, output the error code and message.
if (harness_status.status != 0) {
resultStr += `Harness Error. ` +
`harness_status.status = ${harness_status.status} , ` +
`harness_status.message = ${harness_status.message}\n`;
}
// Output failure metrics if there are many.
const resultCounts = countResultTypes(tests);
if (outputDocument.URL.indexOf('://web-platform.test') >= 0 &&
tests.length >= 50 &&
(resultCounts[1] || resultCounts[2] || resultCounts[3])) {
resultStr += failureMetricSummary(resultCounts);
}
resultStr += testOutput(tests);
resultStr += 'Harness: the test ran to completion.\n';
resultsElement.textContent = resultStr;
function done() {
let body = null;
if (outputDocument.body && outputDocument.body.tagName == 'BODY' &&
outputDocument.body.namespaceURI == xhtmlNS) {
body = outputDocument.body;
}
// A temporary workaround since |window.self| property lookup starts
// failing if the frame is detached. |outputDocument| may be an
// ancestor of |self| so clearing |textContent| may detach |self|.
// To get around this, cache window.self now and use the cached
// value.
// TODO(dcheng): Remove this hack after fixing window/self/frames
// lookup in https://crbug.com/618672
const cachedSelf = window.self;
if (cachedSelf.testRunner) {
// The following DOM operations may show console messages. We
// suppress them because they are not related to the running
// test.
testRunner.setDumpConsoleMessages(false);
// Anything in the body isn't part of the output and so should
// be hidden from the text dump.
if (body) {
body.textContent = '';
}
}
// Add the results element to the output document.
if (!body) {
// |outputDocument| might be an SVG document.
if (outputDocument.documentElement) {
outputDocument.documentElement.remove();
}
let html = outputDocument.createElementNS(xhtmlNS, 'html');
outputDocument.appendChild(html);
body = outputDocument.createElementNS(xhtmlNS, 'body');
body.setAttribute('style', 'white-space:pre;');
html.appendChild(body);
}
outputDocument.body.appendChild(resultsElement);
// IFrames running tests should not complete the harness as the parent
// page will.
let shouldCompleteHarness = (window.self == window.top);
if (cachedSelf.testRunner && shouldCompleteHarness) {
testRunner.notifyDone();
}
}
if (didDispatchLoadEvent || outputDocument.readyState != 'loading') {
// This function might not be the last completion callback, and
// another completion callback might generate more results.
// So, we don't dump the results immediately.
setTimeout(done, 0);
} else {
// Parsing the test HTML isn't finished yet.
window.addEventListener('load', done);
}
}
/**
* Returns a directory part relative to WPT root and a basename part of the
* current test. e.g.
* Current test: file:///.../LayoutTests/external/wpt/pointerevents/foobar.html
* Output: "/pointerevents/foobar"
*/
function pathAndBaseNameInWPT() {
const path = location.pathname;
let matches;
if (location.hostname == 'web-platform.test') {
matches = path.match(/^(\/.*)\.html$/);
return matches ? matches[1] : null;
}
matches = path.match(/external\/wpt(\/.*)\.html$/);
return matches ? matches[1] : null;
}
/** Converts the testharness test status into the corresponding string. */
function convertResult(resultStatus) {
let retVal = '';
switch (resultStatus) {
case 0:
retVal = 'PASS';
break;
case 1:
retVal = 'FAIL';
break;
case 2:
retVal = 'TIMEOUT';
break;
case 3:
retVal = 'NOTRUN';
break;
case 4:
retVal = 'PRECONDITION_FAILED';
break;
default:
retVal = 'NOTRUN';
break;
}
return '[' + retVal + ']';
}
function testOutput(tests) {
let testResults = '';
window.tests = tests;
for (let test of tests) {
testResults += resultLine(test);
}
return testResults;
}
function resultLine(test) {
if (test.status == 0) {
return '';
}
let result = `${convertResult(test.status)} ${sanitize(test.name)}`;
// include error message when test result is FAIL or PRECONDITION_FAILED
if (test.message) {
if (test.status == 1 || test.status == 4) {
result += '\n ' + sanitize(test.message).trim();
}
}
return result + '\n';
}
/** Prepares the given text for display in test results. */
function sanitize(text) {
if (!text) {
return '';
}
// Change each '\' to '\\'
text = text.replace(/\\/g, '\\\\');
// Escape null characters, otherwise diff will think the file is binary.
text = text.replace(/\0/g, '\\0');
// Escape some special characters to improve readability of the output.
text = text.replace(/\r/g, '\\r');
text = text.replace(/\n/g, '\\n');
// Replace machine-dependent path with "...".
if (localPathRegExp) {
text = text.replace(localPathRegExp, '...');
}
return text;
}
function countResultTypes(tests) {
const resultCounts = [0, 0, 0, 0];
for (let test of tests) {
resultCounts[test.status]++;
}
return resultCounts;
}
function failureMetricSummary(resultCounts) {
return `Found ${resultCounts[1]} FAIL,` +
` ${resultCounts[2]} TIMEOUT,` +
` ${resultCounts[3]} NOTRUN.\n`;
}
})();