[READ-ONLY] Mirror of https://github.com/vitest-dev/vitest. Next generation testing framework powered by Vite.
vitest.dev
test
testing-tools
vite
feat(browser): support playwright tracing (#8584)
+771
-33
+1
.gitignore
+1
.gitignore
+1
-1
.vscode/settings.json
+1
-1
.vscode/settings.json
···
23
23
// ],
24
24
25
25
"vitest.ignoreWorkspace": true,
26
-
"vitest.configSearchPatternInclude": "test/{core,cli,config}/{vitest,vite}.config.ts",
26
+
"vitest.configSearchPatternInclude": "test/{core,cli,config,browser}/{vitest,vite}.{config.ts,config.unit.mts}",
27
27
"testing.automaticallyOpenTestResults": "neverOpen",
28
28
29
29
// Enable eslint for all supported languages
+5
docs/.vitepress/config.ts
+5
docs/.vitepress/config.ts
···
305
305
link: '/guide/browser/visual-regression-testing',
306
306
docFooterText: 'Visual Regression Testing | Browser Mode',
307
307
},
308
+
{
309
+
text: 'Trace Viewer',
310
+
link: '/guide/browser/trace-viewer',
311
+
docFooterText: 'Trace Viewer | Browser Mode',
312
+
},
308
313
],
309
314
},
310
315
{
+9
-4
docs/advanced/runner.md
+9
-4
docs/advanced/runner.md
···
27
27
/**
28
28
* Called before running a single test. Doesn't have "result" yet.
29
29
*/
30
-
onBeforeRunTask?: (test: TaskPopulated) => unknown
30
+
onBeforeRunTask?: (test: Test) => unknown
31
31
/**
32
32
* Called before actually running the test function. Already has "result" with "state" and "startTime".
33
33
*/
34
-
onBeforeTryTask?: (test: TaskPopulated, options: { retry: number; repeats: number }) => unknown
34
+
onBeforeTryTask?: (test: Test, options: { retry: number; repeats: number }) => unknown
35
35
/**
36
36
* Called after result and state are set.
37
37
*/
38
-
onAfterRunTask?: (test: TaskPopulated) => unknown
38
+
onAfterRunTask?: (test: Test) => unknown
39
39
/**
40
40
* Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws.
41
41
*/
42
-
onAfterTryTask?: (test: TaskPopulated, options: { retry: number; repeats: number }) => unknown
42
+
onAfterTryTask?: (test: Test, options: { retry: number; repeats: number }) => unknown
43
+
/**
44
+
* Called after the retry resolution happend. Unlike `onAfterTryTask`, the test now has a new state.
45
+
* All `after` hooks were also called by this point.
46
+
*/
47
+
onAfterRetryTask?: (test: Test, options: { retry: number; repeats: number }) => unknown
43
48
44
49
/**
45
50
* Called before running a single suite. Doesn't have "result" yet.
+1
-1
docs/config/index.md
+1
-1
docs/config/index.md
···
2439
2439
2440
2440
Always print console traces when calling any `console` method. This is useful for debugging.
2441
2441
2442
-
### attachmentsDir <Version>3.2.0</Version>
2442
+
### attachmentsDir <Version>3.2.0</Version> {#attachmentsdir}
2443
2443
2444
2444
- **Type:** `string`
2445
2445
- **Default:** `'.vitest-attachments'`
+7
docs/guide/cli-generated.md
+7
docs/guide/cli-generated.md
···
376
376
377
377
Control if Vitest catches uncaught exceptions so they can be reported (default: `true`)
378
378
379
+
### browser.trace
380
+
381
+
- **CLI:** `--browser.trace <mode>`
382
+
- **Config:** [browser.trace](/guide/browser/config#browser-trace)
383
+
384
+
Enable trace view mode. Supported: "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure".
385
+
379
386
### pool
380
387
381
388
- **CLI:** `--pool <pool>`
+47
-3
docs/guide/browser/config.md
+47
-3
docs/guide/browser/config.md
···
128
128
129
129
Configure options for Vite server that serves code in the browser. Does not affect [`test.api`](#api) option. By default, Vitest assigns port `63315` to avoid conflicts with the development server, allowing you to run both in parallel.
130
130
131
-
## browser.provider <Badge type="danger">advanced</Badge> {#browser-provider}
131
+
## browser.provider {#browser-provider}
132
132
133
133
- **Type:** `BrowserProviderOption`
134
134
- **Default:** `'preview'`
···
154
154
155
155
To configure how provider initializes the browser, you can pass down options to the factory function:
156
156
157
-
```ts{7-15,22-27}
157
+
```ts{7-13,20-26}
158
158
import { playwright } from '@vitest/browser/providers/playwright'
159
159
160
160
export default defineConfig({
···
188
188
})
189
189
```
190
190
191
-
### Custom Provider
191
+
### Custom Provider <Badge type="danger">advanced</Badge>
192
192
193
193
::: danger ADVANCED API
194
194
The custom provider API is highly experimental and can change between patches. If you just need to run tests in a browser, use the [`browser.instances`](#browser-instances) option instead.
···
305
305
306
306
::: info
307
307
This is the time it should take for the browser to establish the WebSocket connection with the Vitest server. In normal circumstances, this timeout should never be reached.
308
+
:::
309
+
310
+
## browser.trace
311
+
312
+
- **Type:** `'on' | 'off' | 'on-first-retry' | 'on-all-retries' | 'retain-on-failure' | object`
313
+
- **CLI:** `--browser.trace=on`, `--browser.trace=retain-on-failure`
314
+
- **Default:** `'off'`
315
+
316
+
Capture a trace of your browser test runs. You can preview traces with [Playwright Trace Viewer](https://trace.playwright.dev/).
317
+
318
+
This options supports the following values:
319
+
320
+
- `'on'` - capture trace for all tests. (not recommended as it's performance heavy)
321
+
- `'off'` - do not capture traces.
322
+
- `'on-first-retry'` - capture trace only when retrying the test for the first time.
323
+
- `'on-all-retries'` - capture trace on every retry of the test.
324
+
- `'retain-on-failure'` - capture trace only for tests that fail. This will automatically delete traces for tests that pass.
325
+
- `object` - an object with the following shape:
326
+
327
+
```ts
328
+
interface TraceOptions {
329
+
mode: 'on' | 'off' | 'on-first-retry' | 'on-all-retries' | 'retain-on-failure'
330
+
/**
331
+
* The directory where all traces will be stored. By default, Vitest
332
+
* stores all traces in `__traces__` folder close to the test file.
333
+
*/
334
+
tracesDir?: string
335
+
/**
336
+
* Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview.
337
+
* @default true
338
+
*/
339
+
screenshots?: boolean
340
+
/**
341
+
* If this option is true tracing will
342
+
* - capture DOM snapshot on every action
343
+
* - record network activity
344
+
* @default true
345
+
*/
346
+
snapshots?: boolean
347
+
}
348
+
```
349
+
350
+
::: danger WARNING
351
+
This option is supported only by the [**playwright**](/guide/browser/playwright) provider.
308
352
:::
309
353
310
354
## browser.trackUnhandledErrors
+1
-1
docs/guide/browser/playwright.md
+1
-1
docs/guide/browser/playwright.md
···
18
18
19
19
Vitest opens a single page to run all tests in the same file. You can configure the `launch`, `connect` and `context` when calling `playwright` at the top level or inside instances:
20
20
21
-
```ts{7-15,22-27} [vitest.config.js]
21
+
```ts{7-14,21-26} [vitest.config.js]
22
22
import { playwright } from '@vitest/browser/providers/playwright'
23
23
import { defineConfig } from 'vitest/config'
24
24
+74
docs/guide/browser/trace-viewer.md
+74
docs/guide/browser/trace-viewer.md
···
1
+
# Trace Viewer
2
+
3
+
Vitest Browser Mode supports generating Playwright's [trace files](https://playwright.dev/docs/trace-viewer#viewing-remote-traces). To enable tracing, you need to set the [`trace`](/guide/browser/config#browser-trace) option in the `test.browser` configuration.
4
+
5
+
::: warning
6
+
Generating trace files is only available when using the [Playwright provider](/guide/browser/playwright).
7
+
:::
8
+
9
+
::: code-group
10
+
```ts [vitest.config.js]
11
+
import { defineConfig } from 'vitest/config'
12
+
import { playwright } from '@vitest/browser/providers/playwright'
13
+
14
+
export default defineConfig({
15
+
test: {
16
+
browser: {
17
+
provider: playwright(),
18
+
trace: 'on',
19
+
},
20
+
},
21
+
})
22
+
```
23
+
```bash [CLI]
24
+
vitest --browser.trace=on
25
+
```
26
+
:::
27
+
28
+
By default, Vitest will generate a trace file for each test. You can also configure it to only generate traces on test failures by setting `trace` to `'on-first-retry'`, `'on-all-retries'` or `'retain-on-failure'`. The files will be saved in `__traces__` folder next to your test files. The name of the trace includes the project name, the test name, the [`repeats` count and `retry` count](/api/#test-api-reference):
29
+
30
+
```
31
+
chromium-my-test-0-0.trace.zip
32
+
^^^^^^^^ project name
33
+
^^^^^^ test name
34
+
^ repeat count
35
+
^ retry count
36
+
```
37
+
38
+
To change the output directory, you can set the `tracesDir` option in the `test.browser.trace` configuration. This way all traces will be stored in the same directory, grouped by the test file.
39
+
40
+
```ts [vitest.config.js]
41
+
import { defineConfig } from 'vitest/config'
42
+
import { playwright } from '@vitest/browser/providers/playwright'
43
+
44
+
export default defineConfig({
45
+
test: {
46
+
browser: {
47
+
provider: playwright(),
48
+
trace: {
49
+
mode: 'on',
50
+
// the path is relative to the root of the project
51
+
tracesDir: './playwright-traces',
52
+
},
53
+
},
54
+
},
55
+
})
56
+
```
57
+
58
+
The traces are available in reporters as [annotations](/guide/test-annotations). For example, in the HTML reporter, you can find the link to the trace file in the test details.
59
+
60
+
## Preview
61
+
62
+
To open the trace file, you can use the Playwright Trace Viewer. Run the following command in your terminal:
63
+
64
+
```bash
65
+
npx playwright show-trace "path-to-trace-file"
66
+
```
67
+
68
+
This will start the Trace Viewer and load the specified trace file.
69
+
70
+
Alternatively, you can open the Trace Viewer in your browser at https://trace.playwright.dev and upload the trace file there.
71
+
72
+
## Limitations
73
+
74
+
At the moment, Vitest cannot populate the "Sources" tab in the Trace Viewer. This means that while you can see the actions and screenshots captured during the test, you won't be able to view the source code of your tests directly within the Trace Viewer. You will need to refer back to your code editor to see the test implementation.
+5
packages/runner/src/run.ts
+5
packages/runner/src/run.ts
···
382
382
test.onFailed = undefined
383
383
test.onFinished = undefined
384
384
385
+
await runner.onAfterRetryTask?.(test, {
386
+
retry: retryCount,
387
+
repeats: repeatCount,
388
+
})
389
+
385
390
// skipped with new PendingError
386
391
if (test.result?.pending || test.result?.state === 'skip') {
387
392
test.mode = 'skip'
+191
test/browser/specs/playwright-trace.test.ts
+191
test/browser/specs/playwright-trace.test.ts
···
1
+
import { readdirSync, rmSync } from 'node:fs'
2
+
import { resolve } from 'pathe'
3
+
import { afterEach, describe, expect, test } from 'vitest'
4
+
import { provider, runBrowserTests } from './utils'
5
+
6
+
const tracesFolder = resolve(import.meta.dirname, '../fixtures/trace-view/__traces__')
7
+
const basicTestTracesFolder = resolve(tracesFolder, 'basic.test.ts')
8
+
9
+
describe.runIf(provider.name === 'playwright')('playwright tracing', () => {
10
+
afterEach(() => {
11
+
rmSync(tracesFolder, { recursive: true, force: true })
12
+
})
13
+
14
+
test('vitest generates trace files when running with `on`', async () => {
15
+
const { stderr, ctx } = await runBrowserTests({
16
+
root: './fixtures/trace-view',
17
+
browser: {
18
+
trace: 'on',
19
+
},
20
+
includeTaskLocation: true,
21
+
})
22
+
23
+
expect(stderr).toBe('')
24
+
expect(readdirSync(tracesFolder)).toEqual(['basic.test.ts'])
25
+
expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(`
26
+
[
27
+
"chromium-a-single-test-0-0.trace.zip",
28
+
"chromium-nested-suite-suite-test-0-0.trace.zip",
29
+
"chromium-repeated-retried-tests-0-0.trace.zip",
30
+
"chromium-repeated-retried-tests-0-1.trace.zip",
31
+
"chromium-repeated-retried-tests-0-2.trace.zip",
32
+
"chromium-repeated-retried-tests-1-0.trace.zip",
33
+
"chromium-repeated-retried-tests-2-0.trace.zip",
34
+
"chromium-repeated-test-0-0.trace.zip",
35
+
"chromium-repeated-test-1-0.trace.zip",
36
+
"chromium-repeated-test-2-0.trace.zip",
37
+
"chromium-retried-test-0-0.trace.zip",
38
+
"chromium-retried-test-0-1.trace.zip",
39
+
"chromium-retried-test-0-2.trace.zip",
40
+
"firefox-a-single-test-0-0.trace.zip",
41
+
"firefox-nested-suite-suite-test-0-0.trace.zip",
42
+
"firefox-repeated-retried-tests-0-0.trace.zip",
43
+
"firefox-repeated-retried-tests-0-1.trace.zip",
44
+
"firefox-repeated-retried-tests-0-2.trace.zip",
45
+
"firefox-repeated-retried-tests-1-0.trace.zip",
46
+
"firefox-repeated-retried-tests-2-0.trace.zip",
47
+
"firefox-repeated-test-0-0.trace.zip",
48
+
"firefox-repeated-test-1-0.trace.zip",
49
+
"firefox-repeated-test-2-0.trace.zip",
50
+
"firefox-retried-test-0-0.trace.zip",
51
+
"firefox-retried-test-0-1.trace.zip",
52
+
"firefox-retried-test-0-2.trace.zip",
53
+
"webkit-a-single-test-0-0.trace.zip",
54
+
"webkit-nested-suite-suite-test-0-0.trace.zip",
55
+
"webkit-repeated-retried-tests-0-0.trace.zip",
56
+
"webkit-repeated-retried-tests-0-1.trace.zip",
57
+
"webkit-repeated-retried-tests-0-2.trace.zip",
58
+
"webkit-repeated-retried-tests-1-0.trace.zip",
59
+
"webkit-repeated-retried-tests-2-0.trace.zip",
60
+
"webkit-repeated-test-0-0.trace.zip",
61
+
"webkit-repeated-test-1-0.trace.zip",
62
+
"webkit-repeated-test-2-0.trace.zip",
63
+
"webkit-retried-test-0-0.trace.zip",
64
+
"webkit-retried-test-0-1.trace.zip",
65
+
"webkit-retried-test-0-2.trace.zip",
66
+
]
67
+
`)
68
+
69
+
// traces are also stored in attachments so they are visible in all reporters
70
+
const testModules = ctx.state.getTestModules()
71
+
expect(testModules).toHaveLength(3)
72
+
testModules.forEach((testModule) => {
73
+
for (const test of testModule.children.allTests()) {
74
+
if (test.result().state === 'skipped') {
75
+
continue
76
+
}
77
+
78
+
const annotations = test.annotations()
79
+
expect(annotations.length).toBeGreaterThan(0)
80
+
81
+
annotations.forEach((annotation) => {
82
+
expect(annotation.message).toContain('basic.test.ts/')
83
+
expect(annotation.type).toBe('traces')
84
+
expect(annotation.attachment!.contentType).toBe('application/octet-stream')
85
+
expect(annotation.location).toEqual({
86
+
file: test.module.moduleId,
87
+
line: test.location.line,
88
+
column: test.location.column,
89
+
})
90
+
})
91
+
}
92
+
})
93
+
})
94
+
95
+
test('vitest generates trace files when running with `on-all-retries`', async () => {
96
+
const { stderr } = await runBrowserTests({
97
+
root: './fixtures/trace-view',
98
+
browser: {
99
+
trace: 'on-all-retries',
100
+
},
101
+
})
102
+
103
+
expect(stderr).toBe('')
104
+
expect(readdirSync(tracesFolder)).toEqual(['basic.test.ts'])
105
+
expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(`
106
+
[
107
+
"chromium-repeated-retried-tests-0-1.trace.zip",
108
+
"chromium-repeated-retried-tests-0-2.trace.zip",
109
+
"chromium-retried-test-0-1.trace.zip",
110
+
"chromium-retried-test-0-2.trace.zip",
111
+
"firefox-repeated-retried-tests-0-1.trace.zip",
112
+
"firefox-repeated-retried-tests-0-2.trace.zip",
113
+
"firefox-retried-test-0-1.trace.zip",
114
+
"firefox-retried-test-0-2.trace.zip",
115
+
"webkit-repeated-retried-tests-0-1.trace.zip",
116
+
"webkit-repeated-retried-tests-0-2.trace.zip",
117
+
"webkit-retried-test-0-1.trace.zip",
118
+
"webkit-retried-test-0-2.trace.zip",
119
+
]
120
+
`)
121
+
})
122
+
123
+
test('vitest generates trace files when running with `on-first-retries`', async () => {
124
+
const { stderr } = await runBrowserTests({
125
+
root: './fixtures/trace-view',
126
+
browser: {
127
+
trace: 'on-first-retry',
128
+
},
129
+
})
130
+
131
+
expect(stderr).toBe('')
132
+
expect(readdirSync(tracesFolder)).toEqual(['basic.test.ts'])
133
+
expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(`
134
+
[
135
+
"chromium-repeated-retried-tests-0-1.trace.zip",
136
+
"chromium-retried-test-0-1.trace.zip",
137
+
"firefox-repeated-retried-tests-0-1.trace.zip",
138
+
"firefox-retried-test-0-1.trace.zip",
139
+
"webkit-repeated-retried-tests-0-1.trace.zip",
140
+
"webkit-retried-test-0-1.trace.zip",
141
+
]
142
+
`)
143
+
})
144
+
145
+
test('vitest generates trace files when running with `retain-on-failure`', async () => {
146
+
const { stderr } = await runBrowserTests({
147
+
root: './fixtures/trace-view',
148
+
include: ['./*.test.ts', './*.special.ts'],
149
+
browser: {
150
+
trace: 'retain-on-failure',
151
+
},
152
+
})
153
+
154
+
const failingTestTracesFolder = resolve(tracesFolder, 'failing.special.ts')
155
+
156
+
expect(readdirSync(tracesFolder)).toEqual([
157
+
'basic.test.ts',
158
+
'failing.special.ts',
159
+
])
160
+
expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(`[]`)
161
+
expect(readdirSync(failingTestTracesFolder).sort()).toMatchInlineSnapshot(`
162
+
[
163
+
"chromium-fail-0-0.trace.zip",
164
+
"chromium-repeated-fail-0-0.trace.zip",
165
+
"chromium-repeated-fail-1-0.trace.zip",
166
+
"chromium-repeated-fail-2-0.trace.zip",
167
+
"chromium-retried-fail-0-0.trace.zip",
168
+
"chromium-retried-fail-0-1.trace.zip",
169
+
"chromium-retried-fail-0-2.trace.zip",
170
+
"firefox-fail-0-0.trace.zip",
171
+
"firefox-repeated-fail-0-0.trace.zip",
172
+
"firefox-repeated-fail-1-0.trace.zip",
173
+
"firefox-repeated-fail-2-0.trace.zip",
174
+
"firefox-retried-fail-0-0.trace.zip",
175
+
"firefox-retried-fail-0-1.trace.zip",
176
+
"firefox-retried-fail-0-2.trace.zip",
177
+
"webkit-fail-0-0.trace.zip",
178
+
"webkit-repeated-fail-0-0.trace.zip",
179
+
"webkit-repeated-fail-1-0.trace.zip",
180
+
"webkit-repeated-fail-2-0.trace.zip",
181
+
"webkit-retried-fail-0-0.trace.zip",
182
+
"webkit-retried-fail-0-1.trace.zip",
183
+
"webkit-retried-fail-0-2.trace.zip",
184
+
]
185
+
`)
186
+
187
+
// the default reporter outputs attachments
188
+
expect(stderr).toContain('❯ traces')
189
+
expect(stderr).toContain('↳ __traces__/failing.special.ts/')
190
+
})
191
+
})
+8
packages/runner/src/types/runner.ts
+8
packages/runner/src/types/runner.ts
···
106
106
test: Test,
107
107
options: { retry: number; repeats: number }
108
108
) => unknown
109
+
/**
110
+
* Called after the retry resolution happend. Unlike `onAfterTryTask`, the test now has a new state.
111
+
* All `after` hooks were also called by this point.
112
+
*/
113
+
onAfterRetryTask?: (
114
+
test: Test,
115
+
options: { retry: number; repeats: number }
116
+
) => unknown
109
117
110
118
/**
111
119
* Called before running a single suite. Doesn't have "result" yet.
+3
-2
packages/vitest/src/node/printError.ts
+3
-2
packages/vitest/src/node/printError.ts
···
164
164
}
165
165
printErrorMessage(e, logger)
166
166
if (options.screenshotPaths?.length) {
167
-
const length = options.screenshotPaths.length
167
+
const uniqueScreenshots = Array.from(new Set(options.screenshotPaths))
168
+
const length = uniqueScreenshots.length
168
169
logger.error(`\nFailure screenshot${length > 1 ? 's' : ''}:`)
169
-
logger.error(options.screenshotPaths.map(p => ` - ${c.dim(relative(process.cwd(), p))}`).join('\n'))
170
+
logger.error(uniqueScreenshots.map(p => ` - ${c.dim(relative(process.cwd(), p))}`).join('\n'))
170
171
if (!e.diff) {
171
172
logger.error()
172
173
}
+5
packages/vitest/src/node/state.ts
+5
packages/vitest/src/node/state.ts
···
211
211
return this.reportedTasksMap.get(task)
212
212
}
213
213
214
+
getReportedEntityById(taskId: string): TestModule | TestCase | TestSuite | undefined {
215
+
const task = this.idMap.get(taskId)
216
+
return task ? this.reportedTasksMap.get(task) : undefined
217
+
}
218
+
214
219
updateTasks(packs: TaskResultPack[]): void {
215
220
for (const [id, result, meta] of packs) {
216
221
const task = this.idMap.get(id)
+5
-2
packages/vitest/src/node/test-run.ts
+5
-2
packages/vitest/src/node/test-run.ts
···
16
16
import type { TestRunEndReason } from './types/reporter'
17
17
import assert from 'node:assert'
18
18
import { createHash } from 'node:crypto'
19
+
import { existsSync } from 'node:fs'
19
20
import { copyFile, mkdir } from 'node:fs/promises'
20
21
import { isPrimitive } from '@vitest/utils/helpers'
21
22
import { serializeValue } from '@vitest/utils/serialize'
22
23
import { parseErrorStacktrace } from '@vitest/utils/source-map'
23
24
import mime from 'mime/lite'
24
-
import { basename, dirname, extname, resolve } from 'pathe'
25
+
import { basename, extname, resolve } from 'pathe'
25
26
26
27
export class TestRun {
27
28
constructor(private vitest: Vitest) {}
···
226
227
project.config.attachmentsDir,
227
228
`${sanitizeFilePath(annotation.message)}-${hash}${extname(currentPath)}`,
228
229
)
229
-
await mkdir(dirname(newPath), { recursive: true })
230
+
if (!existsSync(project.config.attachmentsDir)) {
231
+
await mkdir(project.config.attachmentsDir, { recursive: true })
232
+
}
230
233
await copyFile(currentPath, newPath)
231
234
232
235
attachment.path = newPath
+2
packages/vitest/src/runtime/config.ts
+2
packages/vitest/src/runtime/config.ts
···
122
122
// for playwright
123
123
actionTimeout?: number
124
124
}
125
+
trace: BrowserTraceViewMode
125
126
trackUnhandledErrors: boolean
126
127
}
127
128
standalone: boolean
···
162
163
}
163
164
164
165
export type RuntimeOptions = Partial<RuntimeConfig>
166
+
export type BrowserTraceViewMode = 'on' | 'off' | 'on-first-retry' | 'on-all-retries' | 'retain-on-failure'
+31
test/browser/fixtures/trace-view/basic.test.ts
+31
test/browser/fixtures/trace-view/basic.test.ts
···
1
+
import { describe, test } from 'vitest';
2
+
3
+
test('a single test', () => {
4
+
// ...
5
+
})
6
+
7
+
test('repeated test', { repeats: 2 }, () => {
8
+
// ...
9
+
})
10
+
11
+
test('retried test', { retry: 2 }, ({ task }) => {
12
+
if (task.result?.retryCount !== 2) {
13
+
throw new Error('failed test')
14
+
}
15
+
})
16
+
17
+
test('repeated retried tests', { repeats: 2, retry: 2 }, ({ task }) => {
18
+
if (task.result?.retryCount !== 2) {
19
+
throw new Error('failed test')
20
+
}
21
+
})
22
+
23
+
describe('nested suite', () => {
24
+
test('suite test', () => {
25
+
// ...
26
+
})
27
+
28
+
test.skip('skipped test', () => {
29
+
// ...
30
+
})
31
+
})
+13
test/browser/fixtures/trace-view/failing.special.ts
+13
test/browser/fixtures/trace-view/failing.special.ts
+20
test/browser/fixtures/trace-view/vitest.config.ts
+20
test/browser/fixtures/trace-view/vitest.config.ts
···
1
+
import { fileURLToPath } from 'node:url'
2
+
import { defineConfig } from 'vitest/config'
3
+
import { playwright } from '@vitest/browser/providers/playwright'
4
+
5
+
export default defineConfig({
6
+
cacheDir: fileURLToPath(new URL("./node_modules/.vite", import.meta.url)),
7
+
test: {
8
+
browser: {
9
+
enabled: true,
10
+
provider: playwright(),
11
+
instances: [
12
+
{ browser: 'chromium' },
13
+
{ browser: 'firefox' },
14
+
{ browser: 'webkit' },
15
+
],
16
+
headless: true,
17
+
trace: 'off',
18
+
},
19
+
},
20
+
})
+101
-10
packages/browser/src/client/tester/runner.ts
+101
-10
packages/browser/src/client/tester/runner.ts
···
1
-
import type { CancelReason, File, Suite, Task, TaskEventPack, TaskResultPack, Test, TestAnnotation, VitestRunner } from '@vitest/runner'
1
+
import type {
2
+
CancelReason,
3
+
File,
4
+
Suite,
5
+
Task,
6
+
TaskEventPack,
7
+
TaskResultPack,
8
+
Test,
9
+
TestAnnotation,
10
+
VitestRunner,
11
+
} from '@vitest/runner'
2
12
import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest'
3
13
import type { VitestBrowserClientMocker } from './mocker'
14
+
import type { CommandsManager } from './utils'
4
15
import { globalChannel, onCancel } from '@vitest/browser/client'
5
16
import { page, userEvent } from '@vitest/browser/context'
17
+
import { getTestName } from '@vitest/runner/utils'
6
18
import {
7
19
DecodedMap,
8
20
getOriginalPosition,
···
12
24
} from 'vitest/internal/browser'
13
25
import { NodeBenchmarkRunner, VitestTestRunner } from 'vitest/runners'
14
26
import { createStackString, parseStacktrace } from '../../../../utils/src/source-map'
15
-
import { getWorkerState, moduleRunner } from '../utils'
27
+
import { getBrowserState, getWorkerState, moduleRunner } from '../utils'
16
28
import { rpc } from './rpc'
17
29
import { VitestBrowserSnapshotEnvironment } from './snapshot'
18
30
···
43
55
hashMap = browserHashMap
44
56
public sourceMapCache = new Map<string, any>()
45
57
public method = 'run' as TestExecutionMethod
58
+
private commands: CommandsManager
46
59
47
60
constructor(options: BrowserRunnerOptions) {
48
61
super(options.config)
49
62
this.config = options.config
63
+
this.commands = getBrowserState().commands
50
64
}
51
65
52
66
setMethod(method: TestExecutionMethod) {
53
67
this.method = method
54
68
}
55
69
70
+
private traces = new Map<string, string[]>()
71
+
56
72
onBeforeTryTask: VitestRunner['onBeforeTryTask'] = async (...args) => {
57
73
await userEvent.cleanup()
58
74
await super.onBeforeTryTask?.(...args)
75
+
const trace = this.config.browser.trace
76
+
const test = args[0]
77
+
if (trace === 'off') {
78
+
return
79
+
}
80
+
const { retry, repeats } = args[1]
81
+
if (trace === 'on-all-retries' && retry === 0) {
82
+
return
83
+
}
84
+
if (trace === 'on-first-retry' && retry !== 1) {
85
+
return
86
+
}
87
+
let title = getTestName(test)
88
+
if (retry) {
89
+
title += ` (retry x${retry})`
90
+
}
91
+
if (repeats) {
92
+
title += ` (repeat x${repeats})`
93
+
}
94
+
95
+
const name = getTraceName(test, retry, repeats)
96
+
await this.commands.triggerCommand(
97
+
'__vitest_startChunkTrace',
98
+
[{ name, title }],
99
+
)
100
+
}
101
+
102
+
onAfterRetryTask = async (test: Test, { retry, repeats }: { retry: number; repeats: number }) => {
103
+
const trace = this.config.browser.trace
104
+
if (trace === 'off') {
105
+
return
106
+
}
107
+
if (trace === 'on-all-retries' && retry === 0) {
108
+
return
109
+
}
110
+
if (trace === 'on-first-retry' && retry !== 1) {
111
+
return
112
+
}
113
+
const name = getTraceName(test, retry, repeats)
114
+
if (!this.traces.has(test.id)) {
115
+
this.traces.set(test.id, [])
116
+
}
117
+
const traces = this.traces.get(test.id)!
118
+
const { tracePath } = await this.commands.triggerCommand(
119
+
'__vitest_stopChunkTrace',
120
+
[{ name }],
121
+
) as { tracePath: string }
122
+
traces.push(tracePath)
59
123
}
60
124
61
125
onAfterRunTask = async (task: Test) => {
62
126
await super.onAfterRunTask?.(task)
127
+
const trace = this.config.browser.trace
128
+
const traces = this.traces.get(task.id) || []
129
+
if (traces.length) {
130
+
if (trace === 'retain-on-failure' && task.result?.state === 'pass') {
131
+
await this.commands.triggerCommand(
132
+
'__vitest_deleteTracing',
133
+
[{ traces }],
134
+
)
135
+
}
136
+
else {
137
+
await this.commands.triggerCommand(
138
+
'__vitest_annotateTraces',
139
+
[{ testId: task.id, traces }],
140
+
)
141
+
}
142
+
}
63
143
64
144
if (this.config.bail && task.result?.state === 'fail') {
65
145
const previousFailures = await rpc().getCountOfFailedTests()
···
94
174
await Promise.all([
95
175
super.onBeforeRunSuite?.(suite),
96
176
(async () => {
97
-
if ('filepath' in suite) {
98
-
const map = await rpc().getBrowserFileSourceMap(suite.filepath)
99
-
this.sourceMapCache.set(suite.filepath, map)
100
-
const snapshotEnvironment = this.config.snapshotOptions.snapshotEnvironment
101
-
if (snapshotEnvironment instanceof VitestBrowserSnapshotEnvironment) {
102
-
snapshotEnvironment.addSourceMap(suite.filepath, map)
103
-
}
177
+
if (!('filepath' in suite)) {
178
+
return
179
+
}
180
+
const map = await rpc().getBrowserFileSourceMap(suite.filepath)
181
+
this.sourceMapCache.set(suite.filepath, map)
182
+
const snapshotEnvironment = this.config.snapshotOptions.snapshotEnvironment
183
+
if (snapshotEnvironment instanceof VitestBrowserSnapshotEnvironment) {
184
+
snapshotEnvironment.addSourceMap(suite.filepath, map)
104
185
}
105
186
})(),
106
187
])
···
174
255
return rpc().onTaskUpdate(this.method, task, events)
175
256
}
176
257
177
-
importFile = async (filepath: string) => {
258
+
importFile = async (filepath: string, mode: 'collect' | 'setup') => {
178
259
let hash = this.hashMap.get(filepath)
179
260
if (!hash) {
180
261
hash = Date.now().toString()
···
185
266
const prefix = `/${/^\w:/.test(filepath) ? '@fs/' : ''}`
186
267
const query = `browserv=${hash}`
187
268
const importpath = `${prefix}${filepath}?${query}`.replace(/\/+/g, '/')
269
+
// start tracing before the test file is imported
270
+
const trace = this.config.browser.trace
271
+
if (mode === 'collect' && trace !== 'off') {
272
+
await this.commands.triggerCommand('__vitest_startTracing', [])
273
+
}
188
274
try {
189
275
await import(/* @vite-ignore */ importpath)
190
276
}
···
278
364
})
279
365
280
366
await Promise.all(promises)
367
+
}
368
+
369
+
function getTraceName(task: Task, retryCount: number, repeatsCount: number) {
370
+
const name = getTestName(task, '-').replace(/[^a-z0-9]/gi, '-')
371
+
return `${name}-${repeatsCount}-${retryCount}`
281
372
}
+14
packages/browser/src/node/commands/index.ts
+14
packages/browser/src/node/commands/index.ts
···
14
14
import { screenshotMatcher } from './screenshotMatcher'
15
15
import { selectOptions } from './select'
16
16
import { tab } from './tab'
17
+
import {
18
+
annotateTraces,
19
+
deleteTracing,
20
+
startChunkTrace,
21
+
startTracing,
22
+
stopChunkTrace,
23
+
} from './trace'
17
24
import { type } from './type'
18
25
import { upload } from './upload'
19
26
import { viewport } from './viewport'
···
22
29
readFile: readFile as typeof readFile,
23
30
removeFile: removeFile as typeof removeFile,
24
31
writeFile: writeFile as typeof writeFile,
32
+
33
+
// private commands
25
34
__vitest_fileInfo: _fileInfo as typeof _fileInfo,
26
35
__vitest_upload: upload as typeof upload,
27
36
__vitest_click: click as typeof click,
···
39
48
__vitest_cleanup: keyboardCleanup as typeof keyboardCleanup,
40
49
__vitest_viewport: viewport as typeof viewport,
41
50
__vitest_screenshotMatcher: screenshotMatcher as typeof screenshotMatcher,
51
+
__vitest_deleteTracing: deleteTracing as typeof deleteTracing,
52
+
__vitest_startChunkTrace: startChunkTrace as typeof startChunkTrace,
53
+
__vitest_startTracing: startTracing as typeof startTracing,
54
+
__vitest_stopChunkTrace: stopChunkTrace as typeof stopChunkTrace,
55
+
__vitest_annotateTraces: annotateTraces as typeof annotateTraces,
42
56
}
+126
packages/browser/src/node/commands/trace.ts
+126
packages/browser/src/node/commands/trace.ts
···
1
+
import type { BrowserCommandContext } from 'vitest/node'
2
+
import type { BrowserCommand } from '../plugin'
3
+
import { unlink } from 'node:fs/promises'
4
+
import { basename, dirname, relative, resolve } from 'pathe'
5
+
import { PlaywrightBrowserProvider } from '../providers/playwright'
6
+
7
+
export const startTracing: BrowserCommand<[]> = async ({ context, project, provider, sessionId }) => {
8
+
if (provider instanceof PlaywrightBrowserProvider) {
9
+
if (provider.tracingContexts.has(sessionId)) {
10
+
return
11
+
}
12
+
13
+
provider.tracingContexts.add(sessionId)
14
+
const options = project.config.browser!.trace
15
+
await context.tracing.start({
16
+
screenshots: options.screenshots ?? true,
17
+
snapshots: options.snapshots ?? true,
18
+
// currently, PW shows sources in private methods
19
+
sources: false,
20
+
}).catch(() => {
21
+
provider.tracingContexts.delete(sessionId)
22
+
})
23
+
return
24
+
}
25
+
throw new TypeError(`The ${provider.name} provider does not support tracing.`)
26
+
}
27
+
28
+
export const startChunkTrace: BrowserCommand<[{ name: string; title: string }]> = async (
29
+
command,
30
+
{ name, title },
31
+
) => {
32
+
const { provider, sessionId, testPath, context } = command
33
+
if (!testPath) {
34
+
throw new Error(`stopChunkTrace cannot be called outside of the test file.`)
35
+
}
36
+
if (provider instanceof PlaywrightBrowserProvider) {
37
+
if (!provider.tracingContexts.has(sessionId)) {
38
+
await startTracing(command)
39
+
}
40
+
const path = resolveTracesPath(command, name)
41
+
provider.pendingTraces.set(path, sessionId)
42
+
await context.tracing.startChunk({ name, title })
43
+
return
44
+
}
45
+
throw new TypeError(`The ${provider.name} provider does not support tracing.`)
46
+
}
47
+
48
+
export const stopChunkTrace: BrowserCommand<[{ name: string }]> = async (
49
+
context,
50
+
{ name },
51
+
) => {
52
+
if (context.provider instanceof PlaywrightBrowserProvider) {
53
+
const path = resolveTracesPath(context, name)
54
+
context.provider.pendingTraces.delete(path)
55
+
await context.context.tracing.stopChunk({ path })
56
+
return { tracePath: path }
57
+
}
58
+
throw new TypeError(`The ${context.provider.name} provider does not support tracing.`)
59
+
}
60
+
61
+
function resolveTracesPath({ testPath, project }: BrowserCommandContext, name: string) {
62
+
if (!testPath) {
63
+
throw new Error(`This command can only be called inside a test file.`)
64
+
}
65
+
const options = project.config.browser!.trace
66
+
const sanitizedName = `${project.name.replace(/[^a-z0-9]/gi, '-')}-${name}.trace.zip`
67
+
if (options.tracesDir) {
68
+
return resolve(options.tracesDir, sanitizedName)
69
+
}
70
+
const dir = dirname(testPath)
71
+
const base = basename(testPath)
72
+
return resolve(
73
+
dir,
74
+
'__traces__',
75
+
base,
76
+
`${project.name.replace(/[^a-z0-9]/gi, '-')}-${name}.trace.zip`,
77
+
)
78
+
}
79
+
80
+
export const deleteTracing: BrowserCommand<[{ traces: string[] }]> = async (
81
+
context,
82
+
{ traces },
83
+
) => {
84
+
if (!context.testPath) {
85
+
throw new Error(`stopChunkTrace cannot be called outside of the test file.`)
86
+
}
87
+
if (context.provider instanceof PlaywrightBrowserProvider) {
88
+
return Promise.all(
89
+
traces.map(trace => unlink(trace).catch((err) => {
90
+
if (err.code === 'ENOENT') {
91
+
// Ignore the error if the file doesn't exist
92
+
return
93
+
}
94
+
// Re-throw other errors
95
+
throw err
96
+
})),
97
+
)
98
+
}
99
+
100
+
throw new Error(`provider ${context.provider.name} is not supported`)
101
+
}
102
+
103
+
export const annotateTraces: BrowserCommand<[{ traces: string[]; testId: string }]> = async (
104
+
{ project },
105
+
{ testId, traces },
106
+
) => {
107
+
const vitest = project.vitest
108
+
await Promise.all(traces.map((trace) => {
109
+
const entity = vitest.state.getReportedEntityById(testId)
110
+
return vitest._testRun.annotate(testId, {
111
+
message: relative(project.config.root, trace),
112
+
type: 'traces',
113
+
attachment: {
114
+
path: trace,
115
+
contentType: 'application/octet-stream',
116
+
},
117
+
location: entity?.location
118
+
? {
119
+
file: entity.module.moduleId,
120
+
line: entity.location.line,
121
+
column: entity.location.column,
122
+
}
123
+
: undefined,
124
+
})
125
+
}))
126
+
}
+28
-3
packages/browser/src/node/providers/playwright.ts
+28
-3
packages/browser/src/node/providers/playwright.ts
···
39
39
* The options passed down to [`playwright.connect`](https://playwright.dev/docs/api/class-browsertype#browser-type-launch) method.
40
40
* @see {@link https://playwright.dev/docs/api/class-browsertype#browser-type-launch}
41
41
*/
42
-
launchOptions?: LaunchOptions
42
+
launchOptions?: Omit<
43
+
LaunchOptions,
44
+
'tracesDir'
45
+
>
43
46
/**
44
47
* The options passed down to [`playwright.connect`](https://playwright.dev/docs/api/class-browsertype#browser-type-connect) method.
45
48
*
···
91
94
private browserPromise: Promise<Browser> | null = null
92
95
private closing = false
93
96
97
+
public tracingContexts: Set<string> = new Set()
98
+
public pendingTraces: Map<string, string> = new Map()
99
+
94
100
constructor(
95
101
private project: TestProject,
96
102
private options: PlaywrightProviderOptions,
97
103
) {
98
104
this.browserName = project.config.browser.name as PlaywrightBrowser
99
105
this.mocker = this.createMocker()
106
+
107
+
// make sure the traces are finished if the test hangs
108
+
process.on('SIGTERM', () => {
109
+
if (!this.browser) {
110
+
return
111
+
}
112
+
const promises = []
113
+
for (const [trace, contextId] of this.pendingTraces.entries()) {
114
+
promises.push(() => {
115
+
const context = this.contexts.get(contextId)
116
+
return context?.tracing.stopChunk({ path: trace })
117
+
})
118
+
}
119
+
return Promise.allSettled(promises)
120
+
})
100
121
}
101
122
102
123
private async openBrowser() {
···
131
152
return this.browser
132
153
}
133
154
134
-
const launchOptions = {
155
+
const launchOptions: LaunchOptions = {
135
156
...this.options.launchOptions,
136
157
headless: options.headless,
137
-
} satisfies LaunchOptions
158
+
}
159
+
160
+
if (typeof options.trace === 'object' && options.trace.tracesDir) {
161
+
launchOptions.tracesDir = options.trace?.tracesDir
162
+
}
138
163
139
164
if (this.project.config.inspector.enabled) {
140
165
// NodeJS equivalent defaults: https://nodejs.org/en/learn/getting-started/debugging#enable-inspector
+8
packages/vitest/src/node/cli/cli-config.ts
+8
packages/vitest/src/node/cli/cli-config.ts
···
418
418
trackUnhandledErrors: {
419
419
description: 'Control if Vitest catches uncaught exceptions so they can be reported (default: `true`)',
420
420
},
421
+
trace: {
422
+
description: 'Enable trace view mode. Supported: "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure".',
423
+
argument: '<mode>',
424
+
subcommands: null, // don't support subcommands
425
+
transform(value) {
426
+
return { mode: value }
427
+
},
428
+
},
421
429
orchestratorScripts: null,
422
430
commands: null,
423
431
viewport: null,
+10
packages/vitest/src/node/config/resolveConfig.ts
+10
packages/vitest/src/node/config/resolveConfig.ts
···
786
786
resolved.includeTaskLocation ??= true
787
787
}
788
788
789
+
if (typeof resolved.browser.trace === 'string' || !resolved.browser.trace) {
790
+
resolved.browser.trace = { mode: resolved.browser.trace || 'off' }
791
+
}
792
+
if (resolved.browser.trace.tracesDir != null) {
793
+
resolved.browser.trace.tracesDir = resolvePath(
794
+
resolved.browser.trace.tracesDir,
795
+
resolved.root,
796
+
)
797
+
}
798
+
789
799
const htmlReporter = toArray(resolved.reporters).some((reporter) => {
790
800
if (Array.isArray(reporter)) {
791
801
return reporter[0] === 'html'
+1
packages/vitest/src/node/config/serializeConfig.ts
+1
packages/vitest/src/node/config/serializeConfig.ts
+18
-5
packages/vitest/src/node/reporters/base.ts
+18
-5
packages/vitest/src/node/reporters/base.ts
···
1
-
import type { File, Task } from '@vitest/runner'
1
+
import type { File, Task, TestAnnotation } from '@vitest/runner'
2
2
import type { SerializedError } from '@vitest/utils'
3
3
import type { TestError, UserConsoleLog } from '../../types/general'
4
4
import type { Vitest } from '../core'
···
275
275
276
276
const PADDING = ' '.repeat(padding)
277
277
278
-
annotations.forEach(({ location, type, message }) => {
278
+
const groupedAnnotations: Record<string, TestAnnotation[]> = {}
279
+
280
+
annotations.forEach((annotation) => {
281
+
const { location, type } = annotation
282
+
let group: string
279
283
if (location) {
280
284
const file = relative(test.project.config.root, location.file)
281
-
this[console](`${PADDING}${c.blue(F_POINTER)} ${c.gray(`${file}:${location.line}:${location.column}`)} ${c.bold(type)}`)
285
+
group = `${c.gray(`${file}:${location.line}:${location.column}`)} ${c.bold(type)}`
282
286
}
283
287
else {
284
-
this[console](`${PADDING}${c.blue(F_POINTER)} ${c.bold(type)}`)
288
+
group = c.bold(type)
285
289
}
286
-
this[console](`${PADDING} ${c.blue(F_DOWN_RIGHT)} ${message}`)
290
+
291
+
groupedAnnotations[group] ??= []
292
+
groupedAnnotations[group].push(annotation)
287
293
})
294
+
295
+
for (const group in groupedAnnotations) {
296
+
this[console](`${PADDING}${c.blue(F_POINTER)} ${group}`)
297
+
groupedAnnotations[group].forEach(({ message }) => {
298
+
this[console](`${PADDING} ${c.blue(F_DOWN_RIGHT)} ${message}`)
299
+
})
300
+
}
288
301
}
289
302
290
303
protected getEntityPrefix(entity: TestCase | TestModule | TestSuite): string {
+36
-1
packages/vitest/src/node/types/browser.ts
+36
-1
packages/vitest/src/node/types/browser.ts
···
3
3
import type { Awaitable, ParsedStack, TestError } from '@vitest/utils'
4
4
import type { StackTraceParserOptions } from '@vitest/utils/source-map'
5
5
import type { ViteDevServer } from 'vite'
6
+
import type { BrowserTraceViewMode } from '../../runtime/config'
6
7
import type { BrowserTesterOptions } from '../../types/browser'
7
8
import type { TestProject } from '../project'
8
9
import type { ApiConfig, ProjectConfig } from './config'
···
169
170
}
170
171
171
172
/**
173
+
* Generate traces that can be viewed on https://trace.playwright.dev/
174
+
*
175
+
* This option is supported only by **playwright** provider.
176
+
*/
177
+
trace?: BrowserTraceViewMode | {
178
+
mode: BrowserTraceViewMode
179
+
/**
180
+
* The directory where all traces will be stored. By default, Vitest
181
+
* stores all traces in `__traces__` folder close to the test file.
182
+
*/
183
+
tracesDir?: string
184
+
/**
185
+
* Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview.
186
+
* @default true
187
+
*/
188
+
screenshots?: boolean
189
+
/**
190
+
* If this option is true tracing will
191
+
* - capture DOM snapshot on every action
192
+
* - record network activity
193
+
* @default true
194
+
*/
195
+
snapshots?: boolean
196
+
}
197
+
198
+
/**
172
199
* Directory where screenshots will be saved when page.screenshot() is called
173
200
* If not set, all screenshots are saved to __screenshots__ directory in the same folder as the test file.
174
201
* If this is set, it will be resolved relative to the project root.
···
269
296
parseErrorStacktrace: (error: TestError, options?: StackTraceParserOptions) => ParsedStack[]
270
297
}
271
298
272
-
export interface BrowserCommand<Payload extends unknown[]> {
299
+
export interface BrowserCommand<Payload extends unknown[] = []> {
273
300
(context: BrowserCommandContext, ...payload: Payload): Awaitable<any>
274
301
}
275
302
···
317
344
screenshotFailures: boolean
318
345
locators: {
319
346
testIdAttribute: string
347
+
}
348
+
trace: {
349
+
mode: BrowserTraceViewMode
350
+
tracesDir?: string
351
+
screenshots?: boolean
352
+
snapshots?: boolean
353
+
// TODO: map locations to test ones
354
+
// sources?: boolean
320
355
}
321
356
}
322
357