[READ-ONLY] Mirror of https://github.com/vitest-dev/vitest. Next generation testing framework powered by Vite.
vitest.dev
test
testing-tools
vite
feat(benchmark)!: rewrite the public API (#10113)
+3792
-2125
+1
.gitignore
+1
.gitignore
+1
eslint.config.js
+1
eslint.config.js
+12
-5
pnpm-lock.yaml
+12
-5
pnpm-lock.yaml
···
132
132
strip-literal:
133
133
specifier: ^3.1.0
134
134
version: 3.1.0
135
+
tinybench:
136
+
specifier: ^6.0.1
137
+
version: 6.0.1
135
138
tinyexec:
136
139
specifier: ^1.0.2
137
140
version: 1.0.2
···
835
838
pathe:
836
839
specifier: 'catalog:'
837
840
version: 2.0.3
841
+
tinybench:
842
+
specifier: 'catalog:'
843
+
version: 6.0.1
838
844
839
845
packages/snapshot:
840
846
dependencies:
···
1072
1078
specifier: 'catalog:'
1073
1079
version: 4.0.0-rc.1
1074
1080
tinybench:
1075
-
specifier: ^2.9.0
1076
-
version: 2.9.0
1081
+
specifier: 'catalog:'
1082
+
version: 6.0.1
1077
1083
tinyexec:
1078
1084
specifier: ^1.0.2
1079
1085
version: 1.0.2
···
9709
9715
thread-stream@3.1.0:
9710
9716
resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==}
9711
9717
9712
-
tinybench@2.9.0:
9713
-
resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
9718
+
tinybench@6.0.1:
9719
+
resolution: {integrity: sha512-cMdWsxmysdg8mNWf1pujiWl3TW0cU6m8QuNw55QlnP3I6N96Grb0wnu5N0syHIu3LbiVZCNqlfWzWDq84HZphA==}
9720
+
engines: {node: '>=20.0.0'}
9714
9721
9715
9722
tinyexec@0.3.2:
9716
9723
resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==}
···
19042
19049
dependencies:
19043
19050
real-require: 0.2.0
19044
19051
19045
-
tinybench@2.9.0: {}
19052
+
tinybench@6.0.1: {}
19046
19053
19047
19054
tinyexec@0.3.2: {}
19048
19055
+1
pnpm-workspace.yaml
+1
pnpm-workspace.yaml
+1
-1
.vscode/settings.json
+1
-1
.vscode/settings.json
···
23
23
// ],
24
24
25
25
"vitest.ignoreWorkspace": true,
26
-
"vitest.configSearchPatternInclude": "test/{core,cli,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}",
26
+
"vitest.configSearchPatternInclude": "test/{unit,e2e,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}",
27
27
"testing.automaticallyOpenTestResults": "neverOpen",
28
28
29
29
// Enable eslint for all supported languages
+4
docs/.vitepress/config.ts
+4
docs/.vitepress/config.ts
+3
-230
docs/api/test.md
+3
-230
docs/api/test.md
···
670
670
671
671
## bench <Experimental /> {#bench}
672
672
673
-
- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
673
+
::: warning Updated in Vitest 5
674
+
The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`, and the `bench.skip` / `bench.only` / `bench.todo` helpers have been removed. `bench` is now a [test-context fixture](/guide/test-context#bench) accessed from inside a `test()`.
674
675
675
-
::: danger
676
-
Benchmarking is experimental and does not follow SemVer.
676
+
See the [Benchmarking guide](/guide/benchmarking) for the new API.
677
677
:::
678
-
679
-
`bench` defines a benchmark. In Vitest terms, benchmark is a function that defines a series of operations. Vitest runs this function multiple times to display different performance results.
680
-
681
-
Vitest uses the [`tinybench`](https://github.com/tinylibs/tinybench) library under the hood, inheriting all its options that can be used as a third argument.
682
-
683
-
```ts
684
-
import { bench } from 'vitest'
685
-
686
-
bench('normal sorting', () => {
687
-
const x = [1, 5, 4, 2, 3]
688
-
x.sort((a, b) => {
689
-
return a - b
690
-
})
691
-
}, { time: 1000 })
692
-
```
693
-
694
-
```ts
695
-
export interface Options {
696
-
/**
697
-
* time needed for running a benchmark task (milliseconds)
698
-
* @default 500
699
-
*/
700
-
time?: number
701
-
702
-
/**
703
-
* number of times that a task should run if even the time option is finished
704
-
* @default 10
705
-
*/
706
-
iterations?: number
707
-
708
-
/**
709
-
* function to get the current timestamp in milliseconds
710
-
*/
711
-
now?: () => number
712
-
713
-
/**
714
-
* An AbortSignal for aborting the benchmark
715
-
*/
716
-
signal?: AbortSignal
717
-
718
-
/**
719
-
* Throw if a task fails (events will not work if true)
720
-
*/
721
-
throws?: boolean
722
-
723
-
/**
724
-
* warmup time (milliseconds)
725
-
* @default 100ms
726
-
*/
727
-
warmupTime?: number
728
-
729
-
/**
730
-
* warmup iterations
731
-
* @default 5
732
-
*/
733
-
warmupIterations?: number
734
-
735
-
/**
736
-
* setup function to run before each benchmark task (cycle)
737
-
*/
738
-
setup?: Hook
739
-
740
-
/**
741
-
* teardown function to run after each benchmark task (cycle)
742
-
*/
743
-
teardown?: Hook
744
-
}
745
-
```
746
-
After the test case is run, the output structure information is as follows:
747
-
748
-
```
749
-
name hz min max mean p75 p99 p995 p999 rme samples
750
-
· normal sorting 6,526,368.12 0.0001 0.3638 0.0002 0.0002 0.0002 0.0002 0.0004 ±1.41% 652638
751
-
```
752
-
```ts
753
-
export interface TaskResult {
754
-
/*
755
-
* the last error that was thrown while running the task
756
-
*/
757
-
error?: unknown
758
-
759
-
/**
760
-
* The amount of time in milliseconds to run the benchmark task (cycle).
761
-
*/
762
-
totalTime: number
763
-
764
-
/**
765
-
* the minimum value in the samples
766
-
*/
767
-
min: number
768
-
/**
769
-
* the maximum value in the samples
770
-
*/
771
-
max: number
772
-
773
-
/**
774
-
* the number of operations per second
775
-
*/
776
-
hz: number
777
-
778
-
/**
779
-
* how long each operation takes (ms)
780
-
*/
781
-
period: number
782
-
783
-
/**
784
-
* task samples of each task iteration time (ms)
785
-
*/
786
-
samples: number[]
787
-
788
-
/**
789
-
* samples mean/average (estimate of the population mean)
790
-
*/
791
-
mean: number
792
-
793
-
/**
794
-
* samples variance (estimate of the population variance)
795
-
*/
796
-
variance: number
797
-
798
-
/**
799
-
* samples standard deviation (estimate of the population standard deviation)
800
-
*/
801
-
sd: number
802
-
803
-
/**
804
-
* standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean)
805
-
*/
806
-
sem: number
807
-
808
-
/**
809
-
* degrees of freedom
810
-
*/
811
-
df: number
812
-
813
-
/**
814
-
* critical value of the samples
815
-
*/
816
-
critical: number
817
-
818
-
/**
819
-
* margin of error
820
-
*/
821
-
moe: number
822
-
823
-
/**
824
-
* relative margin of error
825
-
*/
826
-
rme: number
827
-
828
-
/**
829
-
* median absolute deviation
830
-
*/
831
-
mad: number
832
-
833
-
/**
834
-
* p50/median percentile
835
-
*/
836
-
p50: number
837
-
838
-
/**
839
-
* p75 percentile
840
-
*/
841
-
p75: number
842
-
843
-
/**
844
-
* p99 percentile
845
-
*/
846
-
p99: number
847
-
848
-
/**
849
-
* p995 percentile
850
-
*/
851
-
p995: number
852
-
853
-
/**
854
-
* p999 percentile
855
-
*/
856
-
p999: number
857
-
}
858
-
```
859
-
860
-
### bench.skip
861
-
862
-
- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
863
-
864
-
You can use `bench.skip` syntax to skip running certain benchmarks.
865
-
866
-
```ts
867
-
import { bench } from 'vitest'
868
-
869
-
bench.skip('normal sorting', () => {
870
-
const x = [1, 5, 4, 2, 3]
871
-
x.sort((a, b) => {
872
-
return a - b
873
-
})
874
-
})
875
-
```
876
-
877
-
### bench.only
878
-
879
-
- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
880
-
881
-
Use `bench.only` to only run certain benchmarks in a given suite. This is useful when debugging.
882
-
883
-
```ts
884
-
import { bench } from 'vitest'
885
-
886
-
bench.only('normal sorting', () => {
887
-
const x = [1, 5, 4, 2, 3]
888
-
x.sort((a, b) => {
889
-
return a - b
890
-
})
891
-
})
892
-
```
893
-
894
-
### bench.todo
895
-
896
-
- **Type:** `(name: string | Function) => void`
897
-
898
-
Use `bench.todo` to stub benchmarks to be implemented later.
899
-
900
-
```ts
901
-
import { bench } from 'vitest'
902
-
903
-
bench.todo('unimplemented test')
904
-
```
+15
-29
docs/config/benchmark.md
+15
-29
docs/config/benchmark.md
···
9
9
10
10
Options used when running `vitest bench`.
11
11
12
+
## benchmark.enabled
13
+
14
+
- **Type:** `boolean`
15
+
- **Default:** `false`
16
+
17
+
Enables the benchmark project. When set, Vitest creates a dedicated benchmark project alongside your regular test project, runs files matching [`benchmark.include`](#benchmark-include) in it, and exposes the [`bench` fixture](/guide/test-context#bench) to those files. Running `vitest bench` enables this automatically.
18
+
12
19
## benchmark.include
13
20
14
21
- **Type:** `string[]`
···
32
39
33
40
When defined, Vitest will run all matched files with `import.meta.vitest` inside.
34
41
35
-
## benchmark.reporters
42
+
## benchmark.retainSamples
36
43
37
-
- **Type:** `Arrayable<BenchmarkBuiltinReporters | Reporter>`
38
-
- **Default:** `'default'`
44
+
- **Type:** `boolean`
45
+
- **Default:** `false`
39
46
40
-
Custom reporter for output. Can contain one or more built-in report names, reporter instances, and/or paths to custom reporters.
47
+
Include the `samples` array of per-iteration timings on every benchmark result. Disabled by default to reduce memory usage; enable when a custom reporter or API consumer needs the raw samples.
41
48
42
-
## benchmark.outputFile
43
49
44
-
Deprecated in favor of `benchmark.outputJson`.
50
+
## benchmark.suppressExportGetterWarnings
45
51
46
-
## benchmark.outputJson {#benchmark-outputJson}
52
+
- **Type:** `boolean`
53
+
- **Default:** `false`
47
54
48
-
- **Type:** `string | undefined`
49
-
- **Default:** `undefined`
55
+
Suppress the warning printed when a benchmark accesses module export getters too many times. Vitest tracks getter access during benchmark runs because Vite's module runner wraps every export in a getter, and excessive access can dominate the measurement (see [Module Runner Overhead](/guide/benchmarking#module-runner-overhead)). Enable this when you've intentionally accepted the overhead, or when the warning is noisy for benchmarks where the getter cost is negligible.
50
56
51
-
A file path to store the benchmark result, which can be used for `--compare` option later.
52
-
53
-
For example:
54
-
55
-
```sh
56
-
# save main branch's result
57
-
git checkout main
58
-
vitest bench --outputJson main.json
59
-
60
-
# change a branch and compare against main
61
-
git checkout feature
62
-
vitest bench --compare main.json
63
-
```
64
-
65
-
## benchmark.compare {#benchmark-compare}
66
-
67
-
- **Type:** `string | undefined`
68
-
- **Default:** `undefined`
69
-
70
-
A file path to a previous benchmark result to compare against current runs.
+1
-1
docs/config/index.md
+1
-1
docs/config/index.md
···
8
8
9
9
- Create `vitest.config.ts`, which will have the higher priority and will **override** the configuration from `vite.config.ts` (Vitest supports all conventional JS and TS extensions, but doesn't support `json`) - it means all options in your `vite.config` will be **ignored**
10
10
- Pass `--config` option to CLI, e.g. `vitest --config ./path/to/vitest.config.ts`
11
-
- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test`/`benchmark` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests
11
+
- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests
12
12
13
13
To configure `vitest` itself, add `test` property in your Vite config. You'll also need to add a reference to Vitest types using a [triple slash command](https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-types-) at the top of your config file, if you are importing `defineConfig` from `vite` itself.
14
14
-1
docs/config/runner.md
-1
docs/config/runner.md
···
6
6
# runner
7
7
8
8
- **Type:** `VitestRunnerConstructor`
9
-
- **Default:** `node`, when running tests, or `benchmark`, when running benchmarks
10
9
11
10
Path to a custom test runner. This is an advanced feature and should be used with custom library runners. You can read more about it in [the documentation](/api/advanced/runner).
+480
docs/guide/benchmarking.md
+480
docs/guide/benchmarking.md
···
1
+
---
2
+
title: Benchmarking | Guide
3
+
---
4
+
5
+
# Benchmarking
6
+
7
+
Vitest lets you write benchmarks alongside your tests using the `bench` fixture from the [test context](/guide/test-context). Benchmarks are powered by [Tinybench](https://github.com/tinylibs/tinybench) and are defined inside regular `test()` calls, giving you access to the full power of Vitest's test runner: retries, lifecycle hooks, filtering, and assertions.
8
+
9
+
## Defining a Benchmark
10
+
11
+
Use the `bench` fixture to define a benchmark. Call `.run()` to execute it:
12
+
13
+
```ts
14
+
import { expect, test } from 'vitest'
15
+
16
+
test('parsing performance', async ({ bench }) => {
17
+
const result = await bench('parse', () => {
18
+
JSON.parse('{"key":"value"}')
19
+
}).run()
20
+
})
21
+
```
22
+
23
+
The `bench()` function registers a benchmark without executing it. Calling `.run()` runs the benchmark and returns the result. After the test completes, Vitest prints a single-row version of the [comparison table](#comparing-benchmarks) (ops/sec, mean time, percentiles, etc.), so you get the same output for a one-off benchmark as you do for `bench.compare()`.
24
+
25
+
::: warning
26
+
The `bench` fixture is only available in files matched by [`benchmark.include`](/config/#benchmark-include) (default: `**/*.{bench,benchmark}.?(c|m)[jt]s?(x)`). Using `{ bench }` inside a regular test file will throw an error.
27
+
28
+
Whether a file participates in the benchmark run is decided by the filename, not by whether the test uses the `bench` fixture. Renaming `parser.test.ts` to `parser.bench.ts` (or adjusting `benchmark.include`) is what moves it into the benchmark project.
29
+
:::
30
+
31
+
## Running Benchmarks
32
+
33
+
Benchmark files are matched by [`benchmark.include`](/config/#benchmark-include) (default: `**/*.{bench,benchmark}.?(c|m)[jt]s?(x)`) and run in their own project, separate from your regular tests. There are three ways to run them, depending on whether you want to skip them, run them alongside tests, or run them on their own.
34
+
35
+
### `vitest` (default)
36
+
37
+
Without [`benchmark.enabled`](/config/#benchmark-enabled), the `vitest` command only runs regular tests. Benchmark files are ignored entirely. This is the default and the right choice for day-to-day development, since benchmarks are slow and noisy and shouldn't run on every save.
38
+
39
+
### `vitest` with `benchmark.enabled`
40
+
41
+
Set `benchmark.enabled: true` in your config to run benchmarks together with regular tests:
42
+
43
+
```ts [vitest.config.ts]
44
+
import { defineConfig } from 'vitest/config'
45
+
46
+
export default defineConfig({
47
+
test: {
48
+
benchmark: {
49
+
enabled: true,
50
+
},
51
+
},
52
+
})
53
+
```
54
+
55
+
With this config, `vitest` runs your regular tests first, then runs the benchmarks in a separate isolated group (so benchmark execution never overlaps with test execution and adds noise to results). Useful in CI when you want a single command to validate correctness and performance.
56
+
57
+
### `vitest bench`
58
+
59
+
The `bench` subcommand runs only benchmarks and skips regular tests:
60
+
61
+
```bash
62
+
vitest bench
63
+
```
64
+
65
+
This implicitly enables `benchmark.enabled` for the run, so you don't need to set it in the config. Like the `vitest` command, it accepts filename filters and `-t`/`--testNamePattern` to narrow the run:
66
+
67
+
```bash
68
+
# only benchmarks in files matching "parser"
69
+
vitest bench parser
70
+
71
+
# only benchmarks whose test name matches "JSON"
72
+
vitest bench -t JSON
73
+
```
74
+
75
+
## Comparing Benchmarks
76
+
77
+
Use `bench.compare()` to compare multiple benchmarks against each other:
78
+
79
+
```ts
80
+
import { expect, test } from 'vitest'
81
+
82
+
test('compare JSON libraries', async ({ bench }) => {
83
+
const input = '{"key":"value","nested":{"a":1}}'
84
+
85
+
const result = await bench.compare(
86
+
bench('JSON.parse', () => {
87
+
JSON.parse(input)
88
+
}),
89
+
bench('custom parser', () => {
90
+
customParse(input)
91
+
}),
92
+
)
93
+
})
94
+
```
95
+
96
+
When comparing benchmarks, Vitest runs them using interleaved iterations to reduce environmental bias (CPU throttling, GC pressure, etc.) and prints a comparison table after the test completes:
97
+
98
+
<<< ./snippets/benchmark-table.ansi
99
+
100
+
### Options
101
+
102
+
You can pass [options](https://tinylibs.github.io/tinybench/interfaces/BenchOptions.html) as the last argument to `bench.compare()`:
103
+
104
+
```ts
105
+
test('compare with options', async ({ bench }) => {
106
+
const result = await bench.compare(
107
+
bench('lib1', () => { lib1() }),
108
+
bench('lib2', () => { lib2() }),
109
+
{
110
+
iterations: 100,
111
+
time: 1000,
112
+
},
113
+
)
114
+
})
115
+
```
116
+
117
+
You can also pass per-benchmark [options](https://tinylibs.github.io/tinybench/interfaces/FnOptions.html) as the second argument, matching how `test()` accepts options:
118
+
119
+
```ts
120
+
test('benchmarks with setup', async ({ bench }) => {
121
+
const result = await bench.compare(
122
+
bench('with-cache', () => {
123
+
readFromCache()
124
+
}),
125
+
bench(
126
+
'without-cache',
127
+
{ beforeEach: () => clearCache() },
128
+
() => { readFromDisk() },
129
+
),
130
+
)
131
+
})
132
+
```
133
+
134
+
## Comparing Across Projects
135
+
136
+
When your workspace defines multiple projects (e.g., different browsers or runtimes), pass `perProject: true` in the bench options to compare how the same benchmark performs across all of them. Vitest still prints the result inline for the current project, and additionally collects per-project results into a single comparison table at the end of the test run.
137
+
138
+
```ts
139
+
import { test } from 'vitest'
140
+
141
+
test('simple example', async ({ bench }) => {
142
+
await bench('1 + 1', { perProject: true }, () => {
143
+
1 + 1
144
+
}).run()
145
+
})
146
+
```
147
+
148
+
The same test file runs in each project (chromium, firefox, webkit, etc.), and Vitest groups the results:
149
+
150
+
<<< ./snippets/benchmark-per-project.ansi
151
+
152
+
You can also mix `perProject` benchmarks with regular ones inside `bench.compare()`:
153
+
154
+
```ts
155
+
test('compare implementations across browsers', async ({ bench }) => {
156
+
await bench.compare(
157
+
bench('JSON.parse', { perProject: true }, () => {
158
+
JSON.parse('{"key":"value"}')
159
+
}),
160
+
bench('custom parser', () => {
161
+
customParse('{"key":"value"}')
162
+
}),
163
+
)
164
+
})
165
+
```
166
+
167
+
In this case, `custom parser` appears in the normal inline comparison table per project, while `JSON.parse` is additionally collected into the cross-project comparison table at the end.
168
+
169
+
## Asserting Performance
170
+
171
+
Use `toBeFasterThan()` and `toBeSlowerThan()` matchers to assert relative performance between benchmarks:
172
+
173
+
```ts
174
+
import { expect, test } from 'vitest'
175
+
176
+
test('lib1 is faster than lib2', async ({ bench }) => {
177
+
const result = await bench.compare(
178
+
bench('lib1', () => { lib1() }),
179
+
bench('lib2', () => { lib2() }),
180
+
)
181
+
182
+
expect(result.get('lib1')).toBeFasterThan(result.get('lib2'))
183
+
})
184
+
```
185
+
186
+
The `delta` option specifies the minimum relative difference required for the assertion to pass. This helps avoid flaky tests caused by benchmark noise:
187
+
188
+
```ts
189
+
// lib1 must be at least 10% faster than lib2
190
+
expect(result.get('lib1')).toBeFasterThan(result.get('lib2'), {
191
+
delta: 0.1,
192
+
})
193
+
194
+
// lib2 must be at least 20% slower than lib1
195
+
expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'), {
196
+
delta: 0.2,
197
+
})
198
+
```
199
+
200
+
You can also assert absolute performance using standard matchers:
201
+
202
+
```ts
203
+
test('parsing is fast enough', async ({ bench }) => {
204
+
const result = await bench('parse', () => {
205
+
parse(largeInput)
206
+
}).run()
207
+
208
+
expect(result.throughput.mean).toBeGreaterThan(10_000)
209
+
})
210
+
```
211
+
212
+
## Retries
213
+
214
+
Since benchmarks can be noisy, use the `retry` option to automatically retry failing benchmark tests:
215
+
216
+
```ts
217
+
test('performance comparison', { retry: 3 }, async ({ bench }) => {
218
+
const result = await bench.compare(
219
+
bench('lib1', () => { lib1() }),
220
+
bench('lib2', () => { lib2() }),
221
+
)
222
+
223
+
expect(result.get('lib1')).toBeFasterThan(result.get('lib2'))
224
+
})
225
+
```
226
+
227
+
## Storing and Replaying Results
228
+
229
+
Two primitives let you persist benchmark results to disk and compare against them in future runs: the `writeResult` option saves a result, and `bench.from()` reads one back.
230
+
231
+
### `writeResult`
232
+
233
+
Pass `writeResult` as a per-bench option to write the result to a JSON file every time the benchmark runs. The path is resolved against the project root:
234
+
235
+
```ts
236
+
test('parse', async ({ bench }) => {
237
+
await bench(
238
+
'parse',
239
+
{ writeResult: './benchmarks/parse.json' },
240
+
() => parse(largeInput),
241
+
).run()
242
+
})
243
+
```
244
+
245
+
- The benchmark always runs. There is no skip-when-cached behaviour and no CLI flag, the file is overwritten on every successful run.
246
+
- If the function throws, the file is not written.
247
+
- Commit these files alongside your code so reviewers and CI share the same reference points.
248
+
249
+
::: warning
250
+
If you commit these files, keep in mind that benchmark results vary significantly between environments (developer machines, CI runners, different OSes). Designate a single environment (typically CI) to generate the file, and avoid regenerating it locally.
251
+
:::
252
+
253
+
### `bench.from()`
254
+
255
+
`bench.from(name, source)` is a registration that doesn't execute a function. It reads a previously stored result and feeds it into `bench.compare()` (or returns it directly when you call `.run()`).
256
+
257
+
The source can be a path (relative to the project root) or a function that returns the result data, including a Promise:
258
+
259
+
```ts
260
+
test('compare against the stored baseline', async ({ bench }) => {
261
+
const result = await bench.compare(
262
+
bench(
263
+
'current',
264
+
{ writeResult: './benchmarks/parse.json' },
265
+
() => parse(largeInput),
266
+
),
267
+
bench.from('previous', './benchmarks/parse.json'),
268
+
bench.from('remote', () => fetch('https://path/to/external/file.json').then(r => r.json())),
269
+
)
270
+
271
+
expect(result.get('current')).toBeFasterThan(result.get('previous'))
272
+
})
273
+
```
274
+
275
+
You can keep historical artifacts for older versions and compare them against the current implementation. Because `bench.from()` never invokes the function that produced the file, the original benchmark code can be deleted once the artifact is committed:
276
+
277
+
```ts
278
+
test('compare parser versions', async ({ bench }) => {
279
+
const input = '{"key":"value"}'
280
+
281
+
await bench.compare(
282
+
bench.from('v1', './benchmarks/parse.v1.json'),
283
+
bench.from('v2', './benchmarks/parse.v2.json'),
284
+
bench(
285
+
'current',
286
+
{ writeResult: './benchmarks/parse.current.json' },
287
+
() => customParser(input),
288
+
),
289
+
)
290
+
})
291
+
```
292
+
293
+
To produce a new historical artifact, point a fresh `bench()` at that version's implementation, set `writeResult` to a versioned path (`./benchmarks/parse.v3.json`), run it once, then replace the call with `bench.from('v3', './benchmarks/parse.v3.json')`.
294
+
295
+
To regenerate the baseline on demand, gate the write behind an environment variable so the same test either refreshes the artifact or compares against it:
296
+
297
+
```ts
298
+
test('compare parser versions', async ({ bench }) => {
299
+
if (import.meta.env.VITE_WRITE_BENCH) {
300
+
const baseline = bench('baseline', { writeResult: './my-bench.json' }, () => fn())
301
+
await baseline.run()
302
+
}
303
+
else {
304
+
const baseline = bench.from('baseline', './my-bench.json')
305
+
await bench.compare(bench('current', () => fn()), baseline)
306
+
}
307
+
})
308
+
```
309
+
310
+
Run `VITE_WRITE_BENCH=1 vitest bench` to refresh the stored result, and `vitest bench` to compare the current implementation against it.
311
+
312
+
### Per-project artifacts
313
+
314
+
In a multi-project workspace (different browsers, different runtimes), share one benchmark file across projects by including `${projectName}` in the path. The placeholder is substituted with the current project name at write time:
315
+
316
+
```ts
317
+
test('cross-project baseline', async ({ bench }) => {
318
+
await bench(
319
+
'parse',
320
+
// eslint-disable-next-line no-template-curly-in-string
321
+
{ perProject: true, writeResult: './benchmarks/parse.${projectName}.json' },
322
+
() => parse(largeInput),
323
+
).run()
324
+
})
325
+
```
326
+
327
+
Use the same template in `bench.from()` so each project reads its own artifact.
328
+
329
+
## Stability
330
+
331
+
Benchmarks are inherently flaky: CPU load, thermal throttling, GC pressure, and background processes all affect results. Vitest takes several steps to minimize this noise:
332
+
333
+
- **Separate project**: Benchmark files are grouped into their own project based on the [`benchmark.include`](/config/#benchmark-include) pattern. The `bench` fixture is only exposed in files matched by that pattern. Using it inside a regular test file will throw an error.
334
+
- **No concurrency**: Tests within a benchmark file always run sequentially. Benchmark files themselves also run one at a time, never in parallel. This prevents benchmarks from interfering with each other.
335
+
336
+
To further improve stability:
337
+
338
+
- Use the [`retry`](#retries) option to automatically rerun flaky benchmark assertions.
339
+
- Use the [`delta`](#asserting-performance) option in `toBeFasterThan` / `toBeSlowerThan` to allow for acceptable variance.
340
+
- Avoid running benchmarks alongside CPU-intensive processes.
341
+
- Close browsers, IDEs, and other applications that compete for CPU time.
342
+
343
+
### Dead Code Elimination
344
+
345
+
JavaScript engines can optimize away code that has no observable side effects. If your benchmark function doesn't use its result, the engine may skip the computation entirely, producing misleadingly fast numbers:
346
+
347
+
```ts
348
+
test('parsing', async ({ bench }) => {
349
+
// BAD: the engine may eliminate the work
350
+
await bench('parse', () => {
351
+
JSON.parse(input)
352
+
}).run()
353
+
354
+
// GOOD: the result is consumed
355
+
await bench('parse', () => {
356
+
const result = JSON.parse(input)
357
+
doSomething(result)
358
+
}).run()
359
+
})
360
+
```
361
+
362
+
This applies to all engines (V8, JavaScriptCore, SpiderMonkey) but is especially aggressive in V8's TurboFan and JavaScriptCore's FTL compiler tiers.
363
+
364
+
### Module Runner Overhead
365
+
366
+
By default, Vitest runs tests in Node.js using Vite's module runner (configured by [`experimental.viteModuleRunner`](/config/experimental#experimental-vitemodulerunner)). This transforms all module exports into getters, so every access to an imported binding goes through something like `__vite_ssr_module__.value`. In regular tests this overhead is negligible, but in benchmarks where a function is called millions of times, the getter call itself can dominate the measurement.
367
+
368
+
Vitest will print a warning if it detects excessive getter calls (which you can silence via [`benchmark.suppressExportGetterWarnings`](/config/benchmark#benchmark-suppressexportgetterwarnings)), but you should be aware of this when benchmarking imported functions:
369
+
370
+
```ts
371
+
import { parse } from './parser.js'
372
+
373
+
const _parse = parse
374
+
375
+
test('parsing', async ({ bench }) => {
376
+
// BAD: every call to `parse` goes through a getter
377
+
await bench('parse', () => {
378
+
parse(input)
379
+
}).run()
380
+
381
+
// GOOD: store the reference locally to bypass the getter
382
+
await bench('parse', () => {
383
+
_parse(input)
384
+
}).run()
385
+
})
386
+
```
387
+
388
+
If you are the library author, the same overhead applies inside the library you are benchmarking: every cross-module call within its source goes through the same getter wrapper. If you are benchmarking your own library, you have two ways to remove this:
389
+
390
+
**Benchmark the pre-built artifact.** Import the library through its package name (which resolves to its built output) instead of reaching into its source. The built file has already collapsed internal imports into direct references, so Vite's module runner sees a single module with no internal getters:
391
+
392
+
```ts
393
+
// BAD: every internal call inside the library goes through a getter
394
+
import { parse } from '../src/index.ts'
395
+
396
+
// GOOD: the published entry has no internal getters
397
+
import { parse } from 'my-library'
398
+
```
399
+
400
+
If you compare your library against other packages, benchmark the same kind of artifact for every implementation. For workspace packages, make sure the package name resolves to the built output instead of source, for example by externalizing the package in Vite or by importing from `dist`.
401
+
402
+
**Disable the module runner for the benchmark.** If the benchmark does not need Vite transforms, mocks, or Vitest module interception, disable [`experimental.viteModuleRunner`](/config/experimental#experimental-vitemodulerunner) for the benchmark project so Node runs native ESM directly.
403
+
404
+
This only affects Node.js mode. Browser mode uses native ESM imports and does not have this overhead.
405
+
406
+
### Engine-Specific Considerations
407
+
408
+
#### V8 (Node.js, Chrome)
409
+
410
+
- **JIT tiering**: V8 compiles functions through multiple optimization tiers (Sparkplug → Maglev → TurboFan). A function may run at different speeds during warmup vs. steady-state. Tinybench handles warmup automatically, but very short benchmark runs may not reach the highest optimization tier.
411
+
- **Deoptimization**: V8 can "bail out" of optimized code mid-benchmark if it encounters unexpected types or shapes. Keep the types consistent in your benchmark function:
412
+
413
+
```ts
414
+
test('process items', async ({ bench }) => {
415
+
// BAD: mixed shapes cause deoptimization
416
+
await bench('process', () => {
417
+
for (const item of items) {
418
+
// some items have { name: string }, others have { name: string, id: number }
419
+
process(item)
420
+
}
421
+
}).run()
422
+
423
+
// GOOD: consistent object shapes
424
+
await bench('process', () => {
425
+
for (const item of items) {
426
+
// all items have the same shape { name: string, id: number }
427
+
process(item)
428
+
}
429
+
}).run()
430
+
})
431
+
```
432
+
433
+
- **Garbage collection**: Large allocations inside the benchmark loop add GC noise. If you're measuring computation, pre-allocate data in a `setup` hook rather than inside the benchmarked function:
434
+
435
+
```ts
436
+
test('sorting', async ({ bench }) => {
437
+
const original = Array.from({ length: 10000 }, () => Math.random())
438
+
let data: number[]
439
+
440
+
// BAD: allocates a new array every iteration, GC adds noise
441
+
await bench('sort', () => {
442
+
const data = Array.from({ length: 10000 }, () => Math.random())
443
+
data.sort()
444
+
}).run()
445
+
446
+
// GOOD: pre-allocate, copy in beforeEach
447
+
await bench(
448
+
'sort',
449
+
() => { data.sort() },
450
+
{
451
+
beforeEach() {
452
+
data = [...original]
453
+
},
454
+
},
455
+
).run()
456
+
})
457
+
```
458
+
459
+
#### JavaScriptCore (Bun, Safari)
460
+
461
+
- **Different optimization thresholds**: JSC uses its own JIT tiers (LLInt → Baseline → DFG → FTL) with different inlining and optimization heuristics. A benchmark that is fast on V8 may behave very differently on JSC.
462
+
- **Async benchmarks**: Bun's event loop implementation differs from Node.js. If your benchmark involves async operations or timers, results may not be directly comparable across runtimes.
463
+
464
+
#### Browser
465
+
466
+
- **Timer resolution**: Browsers may reduce `performance.now()` precision (e.g., to 100μs or even 1ms) as a security mechanism. This makes very fast operations difficult to measure accurately, so increase iterations to compensate:
467
+
468
+
```ts
469
+
test('fast operations', async ({ bench }) => {
470
+
await bench.compare(
471
+
bench('fast-op', () => { fastOp() }),
472
+
bench('other-op', () => { otherOp() }),
473
+
{
474
+
// more iterations help overcome low timer resolution
475
+
iterations: 1000,
476
+
},
477
+
)
478
+
})
479
+
```
480
+
- **Cross-browser differences**: V8 (Chrome), SpiderMonkey (Firefox), and JSC (Safari) optimize different patterns differently. A benchmark that shows one library winning in Chrome may show the opposite in Firefox.
+1
-1
docs/guide/cli-generated.md
+1
-1
docs/guide/cli-generated.md
+30
docs/guide/migration.md
+30
docs/guide/migration.md
···
13
13
Vitest 5.0 is currently in beta. This section tracks breaking changes as they are merged and may change before the stable release.
14
14
:::
15
15
16
+
### Benchmarking API Rewrite
17
+
18
+
The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`; it is a [test-context fixture](/guide/test-context#bench) accessed from inside a regular `test()`. See the [Benchmarking guide](/guide/benchmarking) for the new API.
19
+
20
+
Removed, with replacements where applicable:
21
+
22
+
- **`bench(name, fn)` at module scope**: destructure `bench` from the test context instead.
23
+
24
+
```ts
25
+
// v4
26
+
import { bench } from 'vitest' // [!code --]
27
+
28
+
bench('sort', () => { // [!code --]
29
+
[3, 1, 2].sort() // [!code --]
30
+
}) // [!code --]
31
+
32
+
// v5
33
+
import { test } from 'vitest' // [!code ++]
34
+
35
+
test('sort', async ({ bench }) => { // [!code ++]
36
+
await bench('sort', () => { [3, 1, 2].sort() }).run() // [!code ++]
37
+
}) // [!code ++]
38
+
```
39
+
40
+
- **`bench.skip`, `bench.only`, `bench.todo`** are removed. Use the regular `test.skip`, `test.only`, `test.todo` on the surrounding `test()` instead.
41
+
- **`benchmark.reporters` / `benchmark.outputFile`** are removed. Benchmark output is now part of the default reporter and the `json` reporter; configure those at the top level via `test.reporters` instead.
42
+
- **`benchmark.compare` config and the `--compare` CLI flag** are removed. Pass [`writeResult`](/guide/benchmarking#storing-and-replaying-results) as a per-bench option to persist a result, and read it back with [`bench.from()`](/guide/benchmarking#bench-from) inside `bench.compare()`.
43
+
- **`benchmark.outputJson` config and the `--outputJson` CLI flag** are removed. Use `--reporter=json --outputFile=<path>` to capture benchmark results; the JSON reporter now includes a `benchmarks` field on each test case.
44
+
- **`Vitest` instance `mode` property** is now always `'test'`. The previous `'benchmark'` value is no longer used; benchmarks run inside a dedicated project of the same `Vitest` instance.
45
+
16
46
### Removed `test.sequential`, `describe.sequential`, and `sequential` Options
17
47
18
48
Vitest 5.0 removes the deprecated `test.sequential`, `describe.sequential`, and `sequential` test options. Use `concurrent: false` when you need a test or suite to opt out of inherited or globally configured concurrency.
+23
docs/guide/test-context.md
+23
docs/guide/test-context.md
···
117
117
}, 2000)
118
118
```
119
119
120
+
### `bench` <Version>5.0.0</Version> {#bench}
121
+
122
+
The `bench` fixture lets you define and run benchmarks inside regular tests. You can measure throughput, compare implementations, and assert relative performance:
123
+
124
+
```ts
125
+
import { expect, test } from 'vitest'
126
+
127
+
test('compare parsers', async ({ bench }) => {
128
+
const result = await bench.compare(
129
+
bench('JSON.parse', () => {
130
+
JSON.parse('{"key":"value"}')
131
+
}),
132
+
bench('custom parser', () => {
133
+
customParse('{"key":"value"}')
134
+
}),
135
+
)
136
+
137
+
expect(result.get('JSON.parse')).toBeFasterThan(result.get('custom parser'))
138
+
})
139
+
```
140
+
141
+
See the [Benchmarks guide](/guide/benchmarking) for full documentation on comparisons, baselines, and assertion matchers.
142
+
120
143
### `onTestFailed`
121
144
122
145
The [`onTestFailed`](/api/hooks#ontestfailed) hook bound to the current test. This API is useful if you are running tests concurrently and need to have a special handling only for this specific test.
+2
-1
packages/runner/package.json
+2
-1
packages/runner/package.json
+1
packages/runner/tsconfig.json
+1
packages/runner/tsconfig.json
+1
-1
packages/vitest/package.json
+1
-1
packages/vitest/package.json
+1
test/browser/package.json
+1
test/browser/package.json
···
9
9
"test:playwright": "PROVIDER=playwright pnpm run test:unit",
10
10
"test:safaridriver": "PROVIDER=webdriverio BROWSER=safari pnpm run test:unit",
11
11
"test-fixtures": "vitest",
12
+
"bench-fixtures": "vitest bench",
12
13
"test-expect-dom": "vitest --root ./fixtures/expect-dom",
13
14
"test-mocking": "vitest --root ./fixtures/mocking",
14
15
"test-logs": "vitest --root ./fixtures/print-logs",
+3
test/browser/vitest.config.mts
+3
test/browser/vitest.config.mts
+2
-1
test/e2e/vitest.config.ts
+2
-1
test/e2e/vitest.config.ts
+1
-1
test/test-utils/cli.ts
+1
-1
test/test-utils/cli.ts
···
86
86
const timeoutId = setTimeout(() => {
87
87
error.message = `Timeout when waiting for error "${expected}".\nReceived:\nstdout: ${this.stdout}\nstderr: ${this.stderr}`
88
88
reject(error)
89
-
}, timeout ?? process.env.CI ? 20_000 : 4_000)
89
+
}, timeout ?? (process.env.CI ? 20_000 : 4_000))
90
90
91
91
const listener = () => {
92
92
if (this[source].includes(expected)) {
+2
-6
test/test-utils/index.ts
+2
-6
test/test-utils/index.ts
···
33
33
printExitCode?: boolean
34
34
preserveAnsi?: boolean
35
35
tty?: boolean
36
-
mode?: 'test' | 'benchmark'
37
36
}
38
37
39
38
export interface RunVitestConfig extends TestUserConfig {
···
128
127
project,
129
128
cliExclude,
130
129
clearScreen,
131
-
compare,
132
-
outputJson,
133
130
mergeReports,
134
131
clearCache,
135
132
// #endregion
···
145
142
;(viteConfig as any).test = rest
146
143
147
144
try {
148
-
ctx = await startVitest(runnerOptions.mode || 'test', cliFilters, {
145
+
ctx = await startVitest(cliFilters, {
149
146
root,
150
147
config: configFile,
151
148
standalone,
···
157
154
project,
158
155
cliExclude,
159
156
clearScreen,
160
-
compare,
161
-
outputJson,
162
157
mergeReports,
163
158
clearCache,
164
159
cache: 'cache' in config ? config.cache : false,
···
172
167
...cliOptions,
173
168
env: {
174
169
NO_COLOR: 'true',
170
+
FORCE_COLOR: undefined,
175
171
AI_AGENT: '',
176
172
...rest.env,
177
173
...cliOptions?.env,
+1
test/unit/package.json
+1
test/unit/package.json
+2
-28
docs/api/advanced/vitest.md
+2
-28
docs/api/advanced/vitest.md
···
5
5
6
6
# Vitest
7
7
8
-
Vitest instance requires the current test mode. It can be either:
8
+
## mode <Deprecated /> {#mode}
9
9
10
-
- `test` when running runtime tests
11
-
- `benchmark` when running benchmarks <Experimental />
12
-
13
-
::: details New in Vitest 4
14
-
Vitest 4 added several new APIs (they are marked with a "4.0.0+" badge) and removed deprecated APIs:
15
-
16
-
- `invalidates`
17
-
- `changedTests` (use [`onFilterWatchedSpecification`](#onfilterwatchedspecification) instead)
18
-
- `server` (use [`vite`](#vite) instead)
19
-
- `getProjectsByTestFile` (use [`getModuleSpecifications`](#getmodulespecifications) instead)
20
-
- `getFileWorkspaceSpecs` (use [`getModuleSpecifications`](#getmodulespecifications) instead)
21
-
- `getModuleProjects` (filter by [`this.projects`](#projects) yourself)
22
-
- `updateLastChanged` (renamed to [`invalidateFile`](#invalidatefile))
23
-
- `globTestSpecs` (use [`globTestSpecifications`](#globtestspecifications) instead)
24
-
- `globTestFiles` (use [`globTestSpecifications`](#globtestspecifications) instead)
25
-
- `listFile` (use [`getRelevantTestSpecifications`](#getrelevanttestspecifications) instead)
26
-
:::
27
-
28
-
## mode
29
-
30
-
### test
31
-
32
-
Test mode will only call functions inside `test` or `it`, and throws an error when `bench` is encountered. This mode uses `include` and `exclude` options in the config to find test files.
33
-
34
-
### benchmark <Experimental /> {#benchmark}
35
-
36
-
Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files.
10
+
Since Vitest 5, this property is always `'test'`.
37
11
38
12
## config
39
13
+2
-4
docs/guide/advanced/index.md
+2
-4
docs/guide/advanced/index.md
···
14
14
15
15
```ts
16
16
function startVitest(
17
-
mode: VitestRunMode,
18
17
cliFilters: string[] = [],
19
18
options: CliOptions = {},
20
19
viteOverrides?: ViteUserConfig,
···
27
26
```js
28
27
import { startVitest } from 'vitest/node'
29
28
30
-
const vitest = await startVitest('test')
29
+
const vitest = await startVitest()
31
30
32
31
await vitest.close()
33
32
```
···
47
46
```ts
48
47
import type { TestModule } from 'vitest/node'
49
48
50
-
const vitest = await startVitest('test')
49
+
const vitest = await startVitest()
51
50
52
51
console.log(vitest.state.getTestModules()) // [TestModule]
53
52
```
···
60
59
61
60
```ts
62
61
function createVitest(
63
-
mode: VitestRunMode,
64
62
options: CliOptions,
65
63
viteOverrides: ViteUserConfig = {},
66
64
vitestOptions: VitestOptions = {},
+7
docs/guide/snippets/benchmark-per-project.ansi
+7
docs/guide/snippets/benchmark-per-project.ansi
···
1
+
[1m Cross-Project Benchmark Comparison[22m
2
+
3
+
[2mtest/parser.bench.ts > regular parsing > parsing[22m
4
+
[1mname [22m [1m hz[22m [1m min[22m [1m max[22m [1m mean[22m [1m p75[22m [1m p99[22m [1m p995[22m [1m p999[22m [1m rme[22m [1msamples[22m
5
+
webkit (bench) [34m6,730,808.15[39m [36m0.0000[39m [36m1.0000[39m [36m0.0001[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [2m±6.20%[22m [2m6731808[22m[1m[32m fastest[39m[22m
6
+
chromium (bench) [34m3,841,599.95[39m [36m0.0000[39m [36m0.2000[39m [36m0.0003[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.1000[39m [2m±1.96%[22m [2m3851571[22m
7
+
firefox (bench) [34m3,546,066.28[39m [36m0.0000[39m [36m5.0000[39m [36m0.0003[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [2m±6.26%[22m [2m3547062[22m[1m[90m slowest[39m[22m
+9
docs/guide/snippets/benchmark-table.ansi
+9
docs/guide/snippets/benchmark-table.ansi
···
1
+
[32m✓[39m [30m[41m bench [49m[39m test/basic.bench.ts [2m([22m[2m1 test[22m[2m)[22m[33m 3446[2mms[22m[39m
2
+
[33m[2m✓[22m[39m different libraries[33m 3445[2mms[22m[39m
3
+
[1mname [22m [1m hz[22m [1m min[22m [1m max[22m [1m mean[22m [1m p75[22m [1m p99[22m [1m p995[22m [1m p999[22m [1m rme[22m [1msamples[22m
4
+
lib 1 [34m40,678,348.31[39m [36m0.0000[39m [36m0.0002[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [2m±0.95%[22m [2m 53989[22m[1m[32m fastest[39m[22m
5
+
lib 3 [34m39,152,132.23[39m [36m0.0000[39m [36m0.0002[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [2m±0.93%[22m [2m 52080[22m
6
+
lib 2 [34m38,088,138.97[39m [36m0.0000[39m [36m0.0066[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [36m0.0000[39m [2m±1.58%[22m [2m 50618[22m[1m[90m slowest[39m[22m
7
+
8
+
[2m Test Files [22m [1m[32m1 passed[39m[22m[90m (1)[39m
9
+
[2m Tests [22m [1m[32m1 passed[39m[22m[90m (1)[39m
+5
-1
packages/browser/src/types.ts
+5
-1
packages/browser/src/types.ts
···
1
1
import type { MockedModuleSerialized, ServerIdResolution, ServerMockResolution } from '@vitest/mocker'
2
-
import type { TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner'
2
+
import type { BaselineData, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner'
3
3
import type { BirpcReturn } from 'birpc'
4
4
import type {
5
5
AfterSuiteRunMeta,
···
8
8
RunnerTestFile,
9
9
SerializedTestSpecification,
10
10
SnapshotResult,
11
+
TestBenchmark,
11
12
TestExecutionMethod,
12
13
UserConsoleLog,
13
14
} from 'vitest'
···
21
22
onCollected: (method: TestExecutionMethod, files: RunnerTestFile[]) => Promise<void>
22
23
onTaskArtifactRecord: <Artifact extends TestArtifact>(testId: string, artifact: Artifact) => Promise<Artifact>
23
24
onTaskUpdate: (method: TestExecutionMethod, packs: TaskResultPack[], events: TaskEventPack[]) => void
25
+
onTestBenchmark: (testId: string, benchmark: TestBenchmark) => void
26
+
readBenchmarkResult: (relativePath: string) => Promise<BaselineData | null>
27
+
writeBenchmarkResult: (relativePath: string, data: BaselineData) => Promise<void>
24
28
onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
25
29
cancelCurrentRun: (reason: CancelReason) => void
26
30
getCountOfFailedTests: () => number
+2
-2
packages/runner/src/collect.ts
+2
-2
packages/runner/src/collect.ts
···
112
112
}
113
113
catch (e) {
114
114
const errors = e instanceof AggregateError
115
-
? e.errors.map(e => processError(e, runner.config.diffOptions))
116
-
: [processError(e, runner.config.diffOptions)]
115
+
? e.errors.map(e => processError(e, runner.config._diffOptions))
116
+
: [processError(e, runner.config._diffOptions)]
117
117
file.result = {
118
118
state: 'fail',
119
119
errors,
+1
packages/runner/src/fixture.ts
+1
packages/runner/src/fixture.ts
+10
-10
packages/runner/src/run.ts
+10
-10
packages/runner/src/run.ts
···
146
146
await Promise.all(hooks.map(fn => limitMaxConcurrency(() => fn(test.context))))
147
147
}
148
148
catch (e) {
149
-
failTask(test.result!, e, runner.config.diffOptions)
149
+
failTask(test.result!, e, runner.config._diffOptions)
150
150
}
151
151
}
152
152
else {
···
155
155
await limitMaxConcurrency(() => fn(test.context))
156
156
}
157
157
catch (e) {
158
-
failTask(test.result!, e, runner.config.diffOptions)
158
+
failTask(test.result!, e, runner.config._diffOptions)
159
159
}
160
160
}
161
161
}
···
662
662
}
663
663
}
664
664
catch (e) {
665
-
failTask(test.result!, e, runner.config.diffOptions)
665
+
failTask(test.result!, e, runner.config._diffOptions)
666
666
}
667
667
668
668
try {
669
669
await runner.onTaskFinished?.(test)
670
670
}
671
671
catch (e) {
672
-
failTask(test.result!, e, runner.config.diffOptions)
672
+
failTask(test.result!, e, runner.config._diffOptions)
673
673
}
674
674
675
675
try {
···
685
685
await callFixtureCleanupFrom(test.context, fixtureCheckpoint)
686
686
}
687
687
catch (e) {
688
-
failTask(test.result!, e, runner.config.diffOptions)
688
+
failTask(test.result!, e, runner.config._diffOptions)
689
689
}
690
690
691
691
if (test.onFinished?.length) {
···
709
709
repeats: repeatCount,
710
710
})
711
711
}).catch((error) => {
712
-
failTask(test.result!, error, runner.config.diffOptions)
712
+
failTask(test.result!, error, runner.config._diffOptions)
713
713
})
714
714
715
715
// Clean up fixtures that were created for aroundEach (before the checkpoint)
···
718
718
await callFixtureCleanup(test.context)
719
719
}
720
720
catch (e) {
721
-
failTask(test.result!, e, runner.config.diffOptions)
721
+
failTask(test.result!, e, runner.config._diffOptions)
722
722
}
723
723
724
724
// skipped with new PendingError
···
886
886
))
887
887
}
888
888
catch (e) {
889
-
failTask(suite.result!, e, runner.config.diffOptions)
889
+
failTask(suite.result!, e, runner.config._diffOptions)
890
890
markTasksAsSkipped(suite, runner)
891
891
return
892
892
}
···
934
934
}
935
935
}
936
936
catch (e) {
937
-
failTask(suite.result!, e, runner.config.diffOptions)
937
+
failTask(suite.result!, e, runner.config._diffOptions)
938
938
}
939
939
}
940
940
})
···
944
944
if (!suiteRan) {
945
945
markTasksAsSkipped(suite, runner)
946
946
}
947
-
failTask(suite.result!, e, runner.config.diffOptions)
947
+
failTask(suite.result!, e, runner.config._diffOptions)
948
948
}
949
949
950
950
if (suite.mode === 'run' || suite.mode === 'queued') {
+1
packages/runner/src/suite.ts
+1
packages/runner/src/suite.ts
+5
packages/runner/src/types.ts
+5
packages/runner/src/types.ts
···
2
2
CancelReason,
3
3
FileSpecification,
4
4
TestTagDefinition,
5
+
TestTryOptions,
5
6
VitestRunner,
6
7
VitestRunnerConfig,
7
8
VitestRunnerConstructor,
···
12
13
AfterEachListener,
13
14
AroundAllListener,
14
15
AroundEachListener,
16
+
BaselineData,
15
17
BeforeAllListener,
16
18
BeforeEachListener,
17
19
BrowserTraceArtifact,
···
40
42
Task,
41
43
TaskBase,
42
44
TaskCustomOptions,
45
+
TaskEventData,
43
46
TaskEventPack,
44
47
TaskHook,
45
48
TaskMeta,
···
58
61
TestArtifactLocation,
59
62
TestArtifactRegistry,
60
63
TestAttachment,
64
+
TestBenchmark,
65
+
TestBenchmarkTask,
61
66
TestContext,
62
67
TestFunction,
63
68
TestOptions,
+7
-5
packages/vitest/src/defaults.ts
+7
-5
packages/vitest/src/defaults.ts
···
14
14
'**/node_modules/**',
15
15
'**/.git/**',
16
16
]
17
-
export const benchmarkConfigDefaults: Required<
18
-
Omit<BenchmarkUserOptions, 'outputFile' | 'compare' | 'outputJson'>
19
-
> = {
17
+
export const benchmarkConfigDefaults: Required<BenchmarkUserOptions> = {
18
+
enabled: false,
20
19
include: ['**/*.{bench,benchmark}.?(c|m)[jt]s?(x)'],
21
20
exclude: defaultExclude,
22
21
includeSource: [],
23
-
reporters: ['default'],
24
-
includeSamples: false,
22
+
retainSamples: false,
23
+
suppressExportGetterWarnings: false,
24
+
// Populated automatically when Vitest clones the parent project; the default
25
+
// here applies to the (unused) raw config that's never run as a benchmark.
26
+
projectName: '',
25
27
}
26
28
27
29
// These are the generic defaults for coverage. Providers may also set some provider specific defaults.
-21
test/browser/specs/benchmark.test.ts
-21
test/browser/specs/benchmark.test.ts
···
1
-
import { expect, test } from 'vitest'
2
-
import { runVitest } from '../../test-utils'
3
-
4
-
const IS_PLAYWRIGHT = process.env.PROVIDER === 'playwright'
5
-
6
-
test('benchmark', async () => {
7
-
const result = await runVitest({ root: 'fixtures/benchmark' }, [], { mode: 'benchmark' })
8
-
expect(result.stderr).toReportNoErrors()
9
-
10
-
if (IS_PLAYWRIGHT) {
11
-
expect(result.stdout).toContain('✓ |chromium| basic.bench.ts > suite-a')
12
-
expect(result.stdout).toContain('✓ |firefox| basic.bench.ts > suite-a')
13
-
expect(result.stdout).toContain('✓ |webkit| basic.bench.ts > suite-a')
14
-
}
15
-
else {
16
-
expect(result.stdout).toContain('✓ |chrome| basic.bench.ts > suite-a')
17
-
expect(result.stdout).toContain('✓ |firefox| basic.bench.ts > suite-a')
18
-
}
19
-
20
-
expect(result.exitCode).toBe(0)
21
-
})
+63
-10
test/browser/specs/runner.test.ts
+63
-10
test/browser/specs/runner.test.ts
···
1
-
import type { JsonTestResults, Vitest } from 'vitest/node'
1
+
import type { TestBenchmark } from 'vitest'
2
+
import type { JsonTestResult, JsonTestResults, Vitest } from 'vitest/node'
2
3
import { readdirSync } from 'node:fs'
3
4
import { readFile } from 'node:fs/promises'
4
5
import { beforeAll, describe, expect, onTestFailed, test } from 'vitest'
···
12
13
let stderr: string
13
14
let stdout: string
14
15
let browserResultJson: JsonTestResults
15
-
let passedTests: any[]
16
-
let failedTests: any[]
16
+
let passedTests: JsonTestResult[]
17
+
let failedTests: JsonTestResult[]
17
18
let vitest: Vitest
18
19
const events: string[] = []
20
+
const emittedBenchmarks: Array<{ projectName: string; testName: string; benchmark: TestBenchmark }> = []
19
21
20
22
beforeAll(async () => {
21
23
({
···
28
30
{
29
31
onBrowserInit(project) {
30
32
events.push(`onBrowserInit ${project.name}`)
33
+
},
34
+
onTestCaseBenchmark(testCase, benchmark) {
35
+
emittedBenchmarks.push({
36
+
projectName: testCase.project.name || '',
37
+
testName: testCase.fullName,
38
+
benchmark,
39
+
})
31
40
},
32
41
},
33
42
'json',
···
47
56
}))
48
57
49
58
const browserResult = await readFile('./browser.json', 'utf-8')
50
-
browserResultJson = JSON.parse(browserResult)
51
-
const getPassed = results => results.filter(result => result.status === 'passed' && !result.message)
52
-
const getFailed = results => results.filter(result => result.status === 'failed')
59
+
browserResultJson = JSON.parse(browserResult) as JsonTestResults
60
+
const getPassed = (results: JsonTestResult[]) => results.filter(result => result.status === 'passed' && !result.message)
61
+
const getFailed = (results: JsonTestResult[]) => results.filter(result => result.status === 'failed')
53
62
passedTests = getPassed(browserResultJson.testResults)
54
63
failedTests = getFailed(browserResultJson.testResults)
55
64
})
···
70
79
.toEqual(vitest.projects.map(() => expect.arrayContaining(runtimeTestFiles)))
71
80
72
81
const testFilesCount = readdirSync('./test')
73
-
.filter(n => n.includes('.test.') || n.includes('.test-d.'))
82
+
.filter(n => n.includes('.test.') || n.includes('.test-d.') || n.includes('.bench.'))
74
83
.length + 1 // 1 is in-source-test
75
84
76
85
expect(browserResultJson.testResults).toHaveLength(testFilesCount * instances.length)
77
86
expect(passedTests).toHaveLength(browserResultJson.testResults.length)
78
87
expect(failedTests).toHaveLength(0)
88
+
})
89
+
90
+
test('benchmarks run in a dedicated `(bench)` project per browser instance', () => {
91
+
const benchProjects = vitest.projects.filter(p => p.name.endsWith('(bench)'))
92
+
expect(benchProjects.map(p => p.name).sort()).toEqual(
93
+
instances.map(({ browser }) => `${browser} (bench)`).sort(),
94
+
)
95
+
})
96
+
97
+
test('perProject benchmarks emit tasks with the perProject flag in every browser', () => {
98
+
const records = emittedBenchmarks.filter(e =>
99
+
e.testName === 'perProject registrations flow through the browser RPC (onTestBenchmark)',
100
+
)
101
+
// the test calls `.run()` twice, so each browser produces 2 benchmark records
102
+
expect(records.length, `perProject emitted: ${records.length}`).toBe(2 * instances.length)
103
+
for (const record of records) {
104
+
expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1)
105
+
const [task] = record.benchmark.tasks
106
+
expect(task.perProject, `missing perProject flag on ${record.projectName}/${task.name}`).toBe(true)
107
+
expect(task.fromStore).toBeUndefined()
108
+
}
109
+
})
110
+
111
+
test('bench.compare emits one benchmark with both registrations ranked', () => {
112
+
const records = emittedBenchmarks.filter(e =>
113
+
e.testName === 'bench.compare resolves a BenchStorage in the browser',
114
+
)
115
+
expect(records.length).toBe(instances.length)
116
+
for (const record of records) {
117
+
expect(record.benchmark.tasks.map(t => t.name).sort(), `unexpected tasks for ${record.projectName}`).toEqual(['a', 'b'])
118
+
expect(record.benchmark.tasks.map(t => t.rank).sort()).toEqual([1, 2])
119
+
}
120
+
})
121
+
122
+
test('writeResult flows through the write-artifact RPC in every browser', () => {
123
+
const records = emittedBenchmarks.filter(e =>
124
+
e.testName === 'writeResult exercises the writeBenchmarkResult RPC round-trip',
125
+
)
126
+
expect(records.length).toBe(instances.length)
127
+
for (const record of records) {
128
+
expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1)
129
+
const [task] = record.benchmark.tasks
130
+
expect(task.name).toBe('with-write')
131
+
}
79
132
})
80
133
81
134
test('tags are collected', () => {
···
110
163
test('runs in-source tests', () => {
111
164
expect(stdout).toContain('src/actions.ts')
112
165
const actionsTest = passedTests.find(t => t.name.includes('/actions.ts'))
113
-
expect(actionsTest).toBeDefined()
166
+
expect.assert(actionsTest)
114
167
expect(actionsTest.assertionResults).toHaveLength(1)
115
168
})
116
169
···
231
284
return
232
285
}
233
286
if (testCase.project.name === 'chromium' || testCase.project.name === 'chrome') {
234
-
expect(testCase.result().errors[0].stacks).toEqual([
287
+
expect(testCase.result().errors?.[0].stacks).toEqual([
235
288
{
236
289
line: 11,
237
290
column: 12,
···
380
433
})
381
434
382
435
const lines = stderr.split('\n')
383
-
const timeoutErrorsIndexes = []
436
+
const timeoutErrorsIndexes: number[] = []
384
437
lines.forEach((line, index) => {
385
438
if (line.includes('TimeoutError:')) {
386
439
timeoutErrorsIndexes.push(index)
+5
-1
test/browser/specs/utils.ts
+5
-1
test/browser/specs/utils.ts
···
43
43
$viteConfig: viteOverrides,
44
44
}, include, runnerOptions)
45
45
46
-
return { ...result, stderr: result.stderr.replace('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.\n', '') }
46
+
return {
47
+
...result,
48
+
ctx: result.ctx!,
49
+
stderr: result.stderr.replace('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.\n', ''),
50
+
}
47
51
}
+43
test/browser/test/browser.bench.ts
+43
test/browser/test/browser.bench.ts
···
1
+
import { expect, test } from 'vitest'
2
+
3
+
// Keep runs tiny — this file smoke-tests the browser RPC path
4
+
// (onTestBenchmark, readBenchmarkResult / writeBenchmarkResult).
5
+
// It is not a measurement harness.
6
+
const fastBenchOptions = {
7
+
time: 0,
8
+
iterations: 2,
9
+
warmupTime: 0,
10
+
warmupIterations: 0,
11
+
}
12
+
13
+
test('perProject registrations flow through the browser RPC (onTestBenchmark)', async ({ bench }) => {
14
+
await bench('1 + 1', { perProject: true, ...fastBenchOptions }, () => {
15
+
const result = 1 + 1
16
+
expect.assert(result === 2)
17
+
}).run()
18
+
await bench('1 + 2', { perProject: true, ...fastBenchOptions }, () => {
19
+
const result = 1 + 2
20
+
expect.assert(result === 3)
21
+
}).run()
22
+
})
23
+
24
+
test('bench.compare resolves a BenchStorage in the browser', async ({ bench }) => {
25
+
const storage = await bench.compare(
26
+
bench('a', () => { const _ = 1 + 1 }),
27
+
bench('b', () => { const _ = 1 + 2 }),
28
+
fastBenchOptions,
29
+
)
30
+
// runtime smoke — every registration is accessible with a valid BenchResult
31
+
expect.assert(typeof storage.get('a').latency.mean === 'number')
32
+
expect.assert(typeof storage.get('b').latency.mean === 'number')
33
+
})
34
+
35
+
test('writeResult exercises the writeBenchmarkResult RPC round-trip', async ({ bench }) => {
36
+
// The browser worker forwards writeResult through the WebSocket RPC to the
37
+
// node side. We don't assert on the file contents here (the spec layer can
38
+
// do that), just that the round-trip completes without throwing.
39
+
const result = await bench('with-write', { writeResult: './out/with-write.json', ...fastBenchOptions }, () => {
40
+
const _ = 1 + 1
41
+
}).run()
42
+
expect.assert(typeof result.latency.mean === 'number')
43
+
})
+103
test/e2e/test/benchmarking.test-d.ts
+103
test/e2e/test/benchmarking.test-d.ts
···
1
+
import type {
2
+
BaselineData,
3
+
Bench,
4
+
BenchFnOptions,
5
+
BenchFromSource,
6
+
BenchRegistration,
7
+
BenchResult,
8
+
BenchStorage,
9
+
TestContext,
10
+
} from 'vitest'
11
+
import { assertType, expect, expectTypeOf, test } from 'vitest'
12
+
13
+
test('plain `bench()` returns BenchRegistration with the name literal narrowed', ({ bench }) => {
14
+
const reg = bench('plain', () => {})
15
+
expectTypeOf(reg).toEqualTypeOf<BenchRegistration<'plain'>>()
16
+
expectTypeOf(reg.name).toEqualTypeOf<'plain'>()
17
+
})
18
+
19
+
test('`bench(name, options, fn)` is the preferred options-second overload', ({ bench }) => {
20
+
const reg = bench('x', { beforeEach: () => {} }, () => {})
21
+
expectTypeOf(reg).toEqualTypeOf<BenchRegistration<'x'>>()
22
+
})
23
+
24
+
test('`bench(name, fn, options)` is a type error — options as third arg is not accepted', ({ bench }) => {
25
+
// @ts-expect-error legacy (name, fn, options) form is no longer accepted
26
+
bench('x', () => {}, { beforeEach: () => {} } satisfies BenchFnOptions)
27
+
})
28
+
29
+
test('`bench.compare(...regs)` returns a BenchStorage keyed by the UNION of registration names', async ({ bench }) => {
30
+
const storage = await bench.compare(bench('a', () => {}), bench('b', () => {}))
31
+
expectTypeOf(storage).toEqualTypeOf<BenchStorage<'a' | 'b'>>()
32
+
expectTypeOf(storage.get('a')).toEqualTypeOf<BenchResult>()
33
+
expectTypeOf(storage.get('b')).toEqualTypeOf<BenchResult>()
34
+
})
35
+
36
+
test('`bench.compare(...).get(unknownName)` is a type error', async ({ bench }) => {
37
+
const storage = await bench.compare(bench('a', () => {}), bench('b', () => {}))
38
+
// @ts-expect-error 'missing' is not one of 'a' | 'b'
39
+
storage.get('missing')
40
+
})
41
+
42
+
test('`bench.compare` accepts BenchCompareOptions as the trailing argument', async ({ bench }) => {
43
+
const storage = await bench.compare(
44
+
bench('a', () => {}),
45
+
bench('b', () => {}),
46
+
{ time: 10, iterations: 5 },
47
+
)
48
+
expectTypeOf(storage).toEqualTypeOf<BenchStorage<'a' | 'b'>>()
49
+
})
50
+
51
+
test('`expect(result).toBeFasterThan` and `.toBeSlowerThan` are callable on BenchResult', () => {
52
+
const a = {} as BenchResult
53
+
const b = {} as BenchResult
54
+
// calling both matchers must type-check; runtime behaviour is in Bucket F
55
+
assertType<void>(expect(a).toBeFasterThan(b))
56
+
assertType<void>(expect(a).toBeFasterThan(b, { delta: 0.1 }))
57
+
assertType<void>(expect(a).toBeSlowerThan(b))
58
+
assertType<void>(expect(a).toBeSlowerThan(b, { delta: 0.1 }))
59
+
})
60
+
61
+
test('`TestContext.bench` is typed as the `Bench` factory', () => {
62
+
type CtxBench = TestContext['bench']
63
+
expectTypeOf<CtxBench>().toEqualTypeOf<Bench>()
64
+
})
65
+
66
+
test('`bench.from(name, path)` returns BenchRegistration with the name literal narrowed', ({ bench }) => {
67
+
const reg = bench.from('baseline', 'results/baseline.json')
68
+
expectTypeOf(reg).toEqualTypeOf<BenchRegistration<'baseline'>>()
69
+
expectTypeOf(reg.name).toEqualTypeOf<'baseline'>()
70
+
})
71
+
72
+
test('`bench.from(name, source)` accepts a function returning BaselineData', ({ bench }) => {
73
+
const sync: BenchFromSource = () => ({} as BaselineData)
74
+
const async: BenchFromSource = async () => ({} as BaselineData)
75
+
expectTypeOf(bench.from('s', sync)).toEqualTypeOf<BenchRegistration<'s'>>()
76
+
expectTypeOf(bench.from('a', async)).toEqualTypeOf<BenchRegistration<'a'>>()
77
+
})
78
+
79
+
test('`bench.from(...).fn` is optional — `bench.from` registrations carry no benchmark function', ({ bench }) => {
80
+
const reg = bench.from('baseline', 'results.json')
81
+
expectTypeOf(reg.fn).toEqualTypeOf<BenchRegistration<'baseline'>['fn']>()
82
+
expectTypeOf(reg.fn).toBeNullable()
83
+
})
84
+
85
+
test('`bench.from` registrations contribute to the name union in `bench.compare`', async ({ bench }) => {
86
+
const storage = await bench.compare(
87
+
bench('live', () => {}),
88
+
bench.from('baseline', 'results.json'),
89
+
)
90
+
expectTypeOf(storage).toEqualTypeOf<BenchStorage<'live' | 'baseline'>>()
91
+
expectTypeOf(storage.get('baseline')).toEqualTypeOf<BenchResult>()
92
+
})
93
+
94
+
test('`bench.from(fn, source)` accepts a function and uses its name as the benchmark name', ({ bench }) => {
95
+
function myBench() {}
96
+
const reg = bench.from(myBench, 'results.json')
97
+
expectTypeOf(reg).toEqualTypeOf<BenchRegistration<string>>()
98
+
})
99
+
100
+
test('`bench.from(name)` without a source is a type error', ({ bench }) => {
101
+
// @ts-expect-error second argument (source) is required
102
+
bench.from('baseline')
103
+
})
+1306
-166
test/e2e/test/benchmarking.test.ts
+1306
-166
test/e2e/test/benchmarking.test.ts
···
1
-
import type { createBenchmarkJsonReport } from 'vitest/src/node/reporters/benchmark/json-formatter.js'
2
-
import fs from 'node:fs'
3
-
import * as pathe from 'pathe'
4
-
import { assert, expect, it } from 'vitest'
5
-
import { runVitest } from '../../test-utils'
1
+
import type { BaselineData, BenchResult, TestBenchmark, TestBenchmarkTask } from 'vitest'
2
+
import type { JsonTestResults } from 'vitest/node'
3
+
import { expect, test } from 'vitest'
4
+
import { runInlineTests } from '../../test-utils'
6
5
7
-
it('sequential', async () => {
8
-
const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/sequential')
9
-
await runVitest({ root }, [], { mode: 'benchmark' })
10
-
const testLog = await fs.promises.readFile(pathe.join(root, 'test.log'), 'utf-8')
11
-
expect(testLog).toMatchSnapshot()
12
-
})
6
+
// Synthetic stats — just enough to satisfy BenchResult shape for the matchers
7
+
// and BaselineData shape for on-disk round-trip tests. All fields beyond
8
+
// `mean` are filled with deterministic numbers so snapshots stay stable.
9
+
function fakeStats(mean: number) {
10
+
return {
11
+
aad: 0,
12
+
critical: 0,
13
+
df: 0,
14
+
mad: 0,
15
+
max: mean,
16
+
samples: undefined,
17
+
mean,
18
+
min: mean,
19
+
moe: 0,
20
+
p50: mean,
21
+
p75: mean,
22
+
p99: mean,
23
+
p995: mean,
24
+
p999: mean,
25
+
rme: 0,
26
+
samplesCount: 2,
27
+
sd: 0,
28
+
sem: 0,
29
+
variance: 0,
30
+
} as const
31
+
}
13
32
14
-
it('summary', async () => {
15
-
const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/reporter')
16
-
const result = await runVitest({ root }, ['summary.bench.ts'], { mode: 'benchmark' })
17
-
expect(result.stderr).toBe('')
18
-
expect(result.stdout).not.toContain('NaNx')
19
-
expect(result.stdout.split('BENCH Summary')[1].replaceAll(/[0-9.]+x/g, '(?)')).toMatchSnapshot()
20
-
})
21
-
22
-
it('non-tty', async () => {
23
-
const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/basic')
24
-
const result = await runVitest({ root }, ['base.bench.ts'], { mode: 'benchmark' })
25
-
const lines = result.stdout.split('\n').slice(4).slice(0, 11)
26
-
const expected = `\
27
-
✓ base.bench.ts > sort
28
-
name
29
-
· normal
30
-
· reverse
31
-
32
-
✓ base.bench.ts > timeout
33
-
name
34
-
· timeout100
35
-
· timeout75
36
-
· timeout50
37
-
· timeout25
38
-
`
39
-
40
-
for (const [index, line] of expected.trim().split('\n').entries()) {
41
-
expect(lines[index]).toMatch(line)
33
+
function fakeBaseline(mean: number): BaselineData {
34
+
return {
35
+
latency: fakeStats(mean),
36
+
throughput: fakeStats(mean > 0 ? 1 / mean : 0),
37
+
period: mean,
38
+
totalTime: mean * 10,
42
39
}
43
-
})
40
+
}
44
41
45
-
it.for([true, false])('includeSamples %s', async (includeSamples) => {
46
-
const result = await runVitest(
42
+
function fakeResult(mean: number): BenchResult {
43
+
return {
44
+
...fakeBaseline(mean),
45
+
name: 'fake',
46
+
} as unknown as BenchResult
47
+
}
48
+
49
+
// keep runs tiny — this suite asserts wiring, not measurement accuracy
50
+
const fastBenchOptions = {
51
+
time: 0,
52
+
iterations: 2,
53
+
warmupTime: 0,
54
+
warmupIterations: 0,
55
+
}
56
+
57
+
test('bench.compare records benchmark results for each registration', async () => {
58
+
const benchmarks: TestBenchmark[] = []
59
+
60
+
const { stderr } = await runInlineTests(
47
61
{
48
-
root: pathe.join(import.meta.dirname, '../fixtures/benchmarking/reporter'),
49
-
benchmark: { includeSamples },
62
+
'basic.bench.ts': /* ts */`
63
+
import { test, inject } from 'vitest'
64
+
65
+
test('compare loops', async ({ bench }) => {
66
+
await bench.compare(
67
+
bench('for', () => { let x = 0; for (let i = 0; i < 10; i++) x += i }),
68
+
bench('while', () => { let x = 0, i = 0; while (i < 10) { x += i; i++ } }),
69
+
inject('options'),
70
+
)
71
+
})
72
+
`,
50
73
},
51
-
['summary.bench.ts'],
52
-
{ mode: 'benchmark' },
74
+
{
75
+
benchmark: { enabled: true },
76
+
reporters: [
77
+
{
78
+
onTestCaseBenchmark(_testCase, benchmark) {
79
+
benchmarks.push(benchmark)
80
+
},
81
+
},
82
+
],
83
+
provide: {
84
+
options: fastBenchOptions,
85
+
},
86
+
},
53
87
)
54
-
assert(result.ctx)
55
-
const allSamples = [...result.ctx.state.idMap.values()]
56
-
.filter(t => t.meta.benchmark)
57
-
.map(t => t.result?.benchmark?.samples)
58
-
if (includeSamples) {
59
-
expect(allSamples[0]).not.toEqual([])
60
-
}
61
-
else {
62
-
expect(allSamples[0]).toEqual([])
63
-
}
88
+
89
+
expect(stderr).toBe('')
90
+
expect(benchmarks).toHaveLength(1)
91
+
const [{ tasks }] = benchmarks
92
+
expect(tasks.map(t => t.name).sort()).toEqual(['for', 'while'])
93
+
expect(tasks.every(t => typeof t.latency.mean === 'number')).toBe(true)
94
+
expect(tasks.map(t => t.rank).sort()).toEqual([1, 2])
64
95
})
65
96
66
-
it('compare', async () => {
67
-
await fs.promises.rm('./fixtures/benchmarking/compare/bench.json', { force: true })
97
+
test('bench accepts options as second argument and rejects them as third', async () => {
98
+
const { stderr, results } = await runInlineTests(
99
+
{
100
+
'sig.bench.ts': /* ts */`
101
+
import { test, expect, inject } from 'vitest'
68
102
69
-
// --outputJson
70
-
{
71
-
const result = await runVitest({
72
-
root: './fixtures/benchmarking/compare',
73
-
outputJson: './bench.json',
74
-
reporters: ['default'],
75
-
}, [], { mode: 'benchmark' })
76
-
expect(result.exitCode).toBe(0)
77
-
expect(fs.existsSync('./fixtures/benchmarking/compare/bench.json')).toBe(true)
78
-
}
103
+
test('bench signatures', async ({ bench }) => {
104
+
const fn = () => 1
105
+
const opts = { async: false }
79
106
80
-
// --compare
81
-
{
82
-
const result = await runVitest({
83
-
root: './fixtures/benchmarking/compare',
84
-
compare: './bench.json',
85
-
reporters: ['default'],
86
-
}, [], { mode: 'benchmark' })
87
-
expect(result.exitCode).toBe(0)
88
-
const lines = result.stdout.split('\n').slice(4).slice(0, 6)
89
-
const expected = `
90
-
✓ basic.bench.ts > suite
91
-
name
92
-
· sleep10
93
-
(baseline)
94
-
· sleep100
95
-
(baseline)
96
-
`
107
+
// options as the 2nd argument (preferred form, matches test())
108
+
const withOpts = bench('with-opts', opts, fn)
109
+
expect(withOpts.name).toBe('with-opts')
110
+
expect(withOpts.fn).toBe(fn)
111
+
expect(withOpts.fnOpts).toBe(opts)
97
112
98
-
for (const [index, line] of expected.trim().split('\n').entries()) {
99
-
expect(lines[index]).toMatch(line.trim())
113
+
// simplest form — no options
114
+
const noOpts = bench('no-opts', fn)
115
+
expect(noOpts.fn).toBe(fn)
116
+
expect(noOpts.fnOpts).toBeUndefined()
117
+
118
+
// legacy (fn, options) form must throw
119
+
expect(() => bench('legacy', fn, opts)).toThrow(/third argument/)
120
+
121
+
// consume the registrations so the unrun-bench warning stays silent
122
+
await bench.compare(withOpts, noOpts, inject('options'))
123
+
})
124
+
`,
125
+
},
126
+
{ benchmark: { enabled: true }, provide: { options: fastBenchOptions } },
127
+
)
128
+
129
+
expect(stderr).toBe('')
130
+
const testCases = [...(results[0]?.children.allTests() ?? [])]
131
+
expect(testCases).toHaveLength(1)
132
+
expect(testCases[0].result()?.state).toBe('passed')
133
+
})
134
+
135
+
test('bench exposes plain and perProject compositions and prints a table', async () => {
136
+
const tasks: TestBenchmarkTask[] = []
137
+
138
+
const { stderr, stdout } = await runInlineTests(
139
+
{
140
+
'compositions.bench.ts': /* ts */`
141
+
import { test, inject } from 'vitest'
142
+
143
+
test('all compositions', async ({ bench }) => {
144
+
await bench.compare(
145
+
bench('plain', () => {}),
146
+
bench('perProject', { perProject: true }, () => {}),
147
+
inject('options'),
148
+
)
149
+
})
150
+
`,
151
+
},
152
+
{
153
+
benchmark: { enabled: true },
154
+
reporters: [
155
+
'default',
156
+
{
157
+
onTestCaseBenchmark(_testCase, benchmark) {
158
+
tasks.push(...benchmark.tasks)
159
+
},
160
+
},
161
+
],
162
+
provide: { options: fastBenchOptions },
163
+
},
164
+
)
165
+
166
+
expect(stderr).toBe('')
167
+
168
+
// every factory shape produces a registration with the right flags
169
+
const byName = Object.fromEntries(
170
+
tasks.map(t => [t.name, { perProject: !!t.perProject }]),
171
+
)
172
+
expect(byName).toEqual({
173
+
plain: { perProject: false },
174
+
perProject: { perProject: true },
175
+
})
176
+
177
+
// snapshot the rendered inline benchmark table. Rows are sorted by name so
178
+
// measurement-driven rank ordering doesn't reshuffle them, and the
179
+
// rank-dependent fastest/slowest suffix is stripped.
180
+
const lines = stdout.split('\n')
181
+
const headerIdx = lines.findIndex(l => /^\s*name\s+hz\s+min/.test(l))
182
+
expect(headerIdx, `inline table header not found in stdout:\n${stdout}`).toBeGreaterThanOrEqual(0)
183
+
const [header, ...rows] = lines.slice(headerIdx, headerIdx + 3)
184
+
const normalized = formatBenchTable([
185
+
header,
186
+
...rows.map(r => r.replace(/\s+(?:fastest|slowest)\s*$/, '')).sort(),
187
+
])
188
+
189
+
expect(normalized).toMatchInlineSnapshot(`
190
+
" name hz min max mean p75 p99 p995 p999 rme samples
191
+
perProject d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+
192
+
plain d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+"
193
+
`)
194
+
195
+
// Only one project ran, so the cross-project section is skipped — a
196
+
// single-row comparison has nothing to compare against.
197
+
expect(stdout).not.toContain('Cross-Project Benchmark Comparison')
198
+
})
199
+
200
+
// Rebuilds a benchmark table with every numeric cell replaced by `d+`, padded
201
+
// with spaces so each column keeps its natural alignment (first column
202
+
// left-aligned, numeric columns right-aligned — same rules the reporter uses).
203
+
// Column widths come from the normalized content so measurement noise at
204
+
// `time: 0` can't shift them between runs. The negative lookbehind on `\d+`
205
+
// keeps column labels like `p75` / `p995` intact.
206
+
function formatBenchTable(tableLines: string[]): string {
207
+
const indent = tableLines[0].match(/^\s*/)![0]
208
+
const rows = tableLines.map(line =>
209
+
line.slice(indent.length).trimEnd().split(/\s{2,}/).map(cell =>
210
+
cell.trim().replace(/(?<![a-z0-9])[\d,.]+/gi, 'd+'),
211
+
),
212
+
)
213
+
const widths = rows[0].map((_, i) =>
214
+
Math.max(...rows.map(r => (r[i] ?? '').length)),
215
+
)
216
+
return rows
217
+
.map(row => indent + row.map((cell, i) =>
218
+
i === 0 ? cell.padEnd(widths[i]) : cell.padStart(widths[i]),
219
+
).join(' '))
220
+
.join('\n')
221
+
}
222
+
223
+
// Runs a single `bench(...).run()` through runInlineTests and pulls apart
224
+
// the reporter output so each test below can snapshot its table in isolation.
225
+
async function runComposition(benchCall: string): Promise<{
226
+
tasks: TestBenchmarkTask[]
227
+
inlineTable: string
228
+
crossProjectSection: string | null
229
+
}> {
230
+
const tasks: TestBenchmarkTask[] = []
231
+
const { stderr, stdout } = await runInlineTests(
232
+
{
233
+
'composition.bench.ts': /* ts */`
234
+
import { test, inject } from 'vitest'
235
+
236
+
test('composition', async ({ bench }) => {
237
+
await ${benchCall}.run(inject('options'))
238
+
})
239
+
`,
240
+
},
241
+
{
242
+
benchmark: { enabled: true },
243
+
reporters: [
244
+
'default',
245
+
{
246
+
onTestCaseBenchmark(_testCase, benchmark) {
247
+
tasks.push(...benchmark.tasks)
248
+
},
249
+
},
250
+
],
251
+
provide: { options: fastBenchOptions },
252
+
},
253
+
)
254
+
255
+
expect(stderr).toBe('')
256
+
257
+
const lines = stdout.split('\n')
258
+
const headerIdx = lines.findIndex(l => /^\s*name\s+hz\s+min/.test(l))
259
+
expect(headerIdx, `inline table header not found in stdout:\n${stdout}`).toBeGreaterThanOrEqual(0)
260
+
const inlineTable = formatBenchTable([
261
+
lines[headerIdx],
262
+
lines[headerIdx + 1].replace(/\s+(?:fastest|slowest)\s*$/, ''),
263
+
])
264
+
265
+
// the cross-project section is a divider + a series of titled sub-tables
266
+
// (each 2 lines: `project …` header + data row). Reformat each sub-table
267
+
// through formatBenchTable while leaving divider and title lines alone.
268
+
let crossProjectSection: string | null = null
269
+
const xpIdx = lines.findIndex(l => /Cross-Project Benchmark Comparison/.test(l))
270
+
if (xpIdx >= 0) {
271
+
const summaryIdx = lines.findIndex((l, i) => i > xpIdx && /^\s*Test Files\s/.test(l))
272
+
const xpLines = lines.slice(xpIdx, summaryIdx < 0 ? undefined : summaryIdx)
273
+
const out: string[] = []
274
+
for (let i = 0; i < xpLines.length; i++) {
275
+
const line = xpLines[i]
276
+
if (/^\s*project\s+hz\s+min/.test(line) && i + 1 < xpLines.length) {
277
+
out.push(formatBenchTable([
278
+
line,
279
+
xpLines[i + 1].replace(/\s+(?:fastest|slowest)\s*$/, ''),
280
+
]))
281
+
i++
282
+
}
283
+
else {
284
+
out.push(line)
285
+
}
100
286
}
287
+
crossProjectSection = out.join('\n').trim()
101
288
}
102
-
})
103
289
104
-
it('basic', { timeout: 60_000 }, async () => {
105
-
const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/basic')
106
-
const benchFile = pathe.join(root, 'bench.json')
107
-
fs.rmSync(benchFile, { force: true })
290
+
return { tasks, inlineTable, crossProjectSection }
291
+
}
108
292
109
-
const result = await runVitest({
110
-
root,
111
-
allowOnly: true,
112
-
outputJson: 'bench.json',
293
+
function assertFlags(task: TestBenchmarkTask, name: string, flags: { perProject?: true }) {
294
+
expect(task.name).toBe(name)
295
+
expect(task.perProject).toBe(flags.perProject)
296
+
}
113
297
114
-
// Verify that type testing cannot be used with benchmark
115
-
typecheck: { enabled: true },
116
-
}, [], { mode: 'benchmark' })
117
-
expect(result.stderr).toBe('')
118
-
expect(result.exitCode).toBe(0)
119
-
120
-
const benchResult = await fs.promises.readFile(benchFile, 'utf-8')
121
-
const resultJson: ReturnType<typeof createBenchmarkJsonReport> = JSON.parse(benchResult)
122
-
const names = resultJson.files.map(f => f.groups.map(g => [g.fullName, g.benchmarks.map(b => b.name)]))
123
-
expect(names).toMatchInlineSnapshot(`
124
-
[
125
-
[
126
-
[
127
-
"base.bench.ts > sort",
128
-
[
129
-
"normal",
130
-
"reverse",
131
-
],
132
-
],
133
-
[
134
-
"base.bench.ts > timeout",
135
-
[
136
-
"timeout100",
137
-
"timeout75",
138
-
"timeout50",
139
-
"timeout25",
140
-
],
141
-
],
142
-
],
143
-
[],
144
-
[
145
-
[
146
-
"only.bench.ts",
147
-
[
148
-
"visited",
149
-
"visited2",
150
-
],
151
-
],
152
-
[
153
-
"only.bench.ts > a0",
154
-
[
155
-
"0",
156
-
],
157
-
],
158
-
[
159
-
"only.bench.ts > a1 > b1 > c1",
160
-
[
161
-
"1",
162
-
],
163
-
],
164
-
[
165
-
"only.bench.ts > a2",
166
-
[
167
-
"2",
168
-
],
169
-
],
170
-
[
171
-
"only.bench.ts > a3 > b3",
172
-
[
173
-
"3",
174
-
],
175
-
],
176
-
[
177
-
"only.bench.ts > a4 > b4",
178
-
[
179
-
"4",
180
-
],
181
-
],
182
-
],
183
-
]
298
+
test('plain `bench()` records a task with no flags', async () => {
299
+
const { tasks, inlineTable, crossProjectSection } = await runComposition(
300
+
`bench('plain', () => {})`,
301
+
)
302
+
expect(tasks).toHaveLength(1)
303
+
assertFlags(tasks[0], 'plain', {})
304
+
expect(crossProjectSection).toBeNull()
305
+
expect(inlineTable).toMatchInlineSnapshot(`
306
+
" name hz min max mean p75 p99 p995 p999 rme samples
307
+
plain d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+"
184
308
`)
185
309
})
310
+
311
+
test('`bench(..., { perProject: true }, fn)` records a perProject task in the inline table; cross-project section is omitted with only one project', async () => {
312
+
const { tasks, inlineTable, crossProjectSection } = await runComposition(
313
+
`bench('perProject', { perProject: true }, () => {})`,
314
+
)
315
+
expect(tasks).toHaveLength(1)
316
+
assertFlags(tasks[0], 'perProject', { perProject: true })
317
+
expect(inlineTable).toMatchInlineSnapshot(`
318
+
" name hz min max mean p75 p99 p995 p999 rme samples
319
+
perProject d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+"
320
+
`)
321
+
expect(crossProjectSection).toBeNull()
322
+
})
323
+
324
+
test('junit reporter embeds the benchmark table inside <system-out>', async () => {
325
+
const { stdout } = await runInlineTests(
326
+
{
327
+
'junit.bench.ts': /* ts */`
328
+
import { test, inject } from 'vitest'
329
+
330
+
test('junit benches', async ({ bench }) => {
331
+
await bench.compare(
332
+
bench('a', () => {}),
333
+
bench('b', () => {}),
334
+
inject('options'),
335
+
)
336
+
})
337
+
`,
338
+
},
339
+
{
340
+
benchmark: { enabled: true },
341
+
reporters: 'junit',
342
+
provide: { options: fastBenchOptions },
343
+
},
344
+
)
345
+
346
+
// extract the <system-out> block from the rendered XML
347
+
// eslint-disable-next-line regexp/no-super-linear-backtracking
348
+
const systemOut = stdout.match(/<system-out>\s*\n([\s\S]*?)<\/system-out>/)?.[1]
349
+
expect(systemOut, stdout).toBeDefined()
350
+
351
+
// a header + 2 data rows — reformat through the shared helper so digits
352
+
// collapse to `d+` and widths become measurement-independent
353
+
const tableLines = systemOut!.split('\n').filter(l => l.trim())
354
+
expect(tableLines).toHaveLength(3)
355
+
const [header, ...rows] = tableLines
356
+
const formatted = formatBenchTable([
357
+
header,
358
+
...rows.map(r => r.replace(/\s+(?:fastest|slowest)\s*$/, '')).sort(),
359
+
])
360
+
361
+
expect(formatted).toMatchInlineSnapshot(`
362
+
"name hz min max mean p75 p99 p995 p999 rme samples
363
+
a d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+
364
+
b d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+"
365
+
`)
366
+
})
367
+
368
+
// Runs the given inner-test source as a virtual bench file and asserts the
369
+
// inner test case finished in `passed` state (i.e. every `expect` inside the
370
+
// bench passed). Lets each outer test push its assertions INTO the inner
371
+
// test file and just verify the aggregate outcome.
372
+
async function runPassingBench(
373
+
filename: string,
374
+
source: string,
375
+
config: Parameters<typeof runInlineTests>[1] = {},
376
+
) {
377
+
const { stderr, results } = await runInlineTests(
378
+
{ [filename]: source },
379
+
{ benchmark: { enabled: true }, provide: { options: fastBenchOptions }, ...config },
380
+
)
381
+
expect(stderr, `stderr should be empty:\n${stderr}`).toBe('')
382
+
const testCases = [...(results[0]?.children.allTests() ?? [])]
383
+
expect(testCases).toHaveLength(1)
384
+
expect(
385
+
testCases[0].result()?.state,
386
+
JSON.stringify(testCases[0].result()?.errors?.map(e => e.message), null, 2),
387
+
).toBe('passed')
388
+
}
389
+
390
+
test('`bench()` inside a non-benchmark project throws a helpful error', async () => {
391
+
const { stderr, results } = await runInlineTests(
392
+
{
393
+
'regular.test.ts': /* ts */`
394
+
import { test, expect } from 'vitest'
395
+
test('misuse', async ({ bench }) => {
396
+
expect(() => bench('x', () => {})).toThrow(
397
+
/Cannot use the \`bench\` test-context fixture within a regular test run/,
398
+
)
399
+
})
400
+
`,
401
+
},
402
+
{ /* benchmark.enabled defaults to false */ },
403
+
)
404
+
expect(stderr).toBe('')
405
+
const testCases = [...(results[0]?.children.allTests() ?? [])]
406
+
expect(testCases).toHaveLength(1)
407
+
expect(testCases[0].result()?.state).toBe('passed')
408
+
})
409
+
410
+
test('`bench.compare()` with zero registrations throws "requires at least 2"', async () => {
411
+
await runPassingBench('zero.bench.ts', /* ts */`
412
+
import { test, expect } from 'vitest'
413
+
test('zero', async ({ bench }) => {
414
+
await expect(bench.compare()).rejects.toThrow(
415
+
/requires at least 2 benchmarks, received 0/,
416
+
)
417
+
})
418
+
`)
419
+
})
420
+
421
+
test('`bench.compare(regA)` with one registration throws and suggests `.run()`', async () => {
422
+
await runPassingBench('one.bench.ts', /* ts */`
423
+
import { test, expect } from 'vitest'
424
+
test('one', async ({ bench }) => {
425
+
await expect(bench.compare(bench('a', () => {}))).rejects.toThrow(
426
+
/received 1.+Consider calling .+bench\\(\\)\\.run\\(\\)/s,
427
+
)
428
+
})
429
+
`)
430
+
})
431
+
432
+
test('`bench.compare(reg, non-reg, reg)` throws the shape error', async () => {
433
+
await runPassingBench('shape.bench.ts', /* ts */`
434
+
import { test, expect } from 'vitest'
435
+
test('shape', async ({ bench }) => {
436
+
await expect(bench.compare(
437
+
bench('a', () => {}),
438
+
{ name: 'fake', fn: () => {} },
439
+
bench('b', () => {}),
440
+
)).rejects.toThrow(
441
+
/expects every argument to be the return value of/,
442
+
)
443
+
})
444
+
`)
445
+
})
446
+
447
+
test('`bench(name, { writeResult }, fn)` writes a result file at the given path', async () => {
448
+
const { stderr, fs } = await runInlineTests(
449
+
{
450
+
'foo.bench.ts': /* ts */`
451
+
import { test, inject } from 'vitest'
452
+
test('write', async ({ bench }) => {
453
+
await bench('x', { writeResult: './out/x.json' }, () => {}).run(inject('options'))
454
+
})
455
+
`,
456
+
},
457
+
{
458
+
benchmark: { enabled: true },
459
+
provide: { options: fastBenchOptions },
460
+
},
461
+
)
462
+
expect(stderr).toBe('')
463
+
const content = JSON.parse(fs.readFile('out/x.json'))
464
+
expect(typeof content.latency.mean).toBe('number')
465
+
expect(typeof content.throughput.mean).toBe('number')
466
+
expect(typeof content.period).toBe('number')
467
+
expect(typeof content.totalTime).toBe('number')
468
+
})
469
+
470
+
test('`writeResult` is overwritten on every successful run', async () => {
471
+
// seed a file with a sentinel value, then run the benchmark — the run
472
+
// must replace the sentinel with fresh measurements
473
+
const sentinel = fakeBaseline(99999)
474
+
const { stderr, fs } = await runInlineTests(
475
+
{
476
+
'upd.bench.ts': /* ts */`
477
+
import { test, inject } from 'vitest'
478
+
test('upd', async ({ bench }) => {
479
+
await bench('x', { writeResult: './out/upd.json' }, () => {}).run(inject('options'))
480
+
})
481
+
`,
482
+
'out/upd.json': JSON.stringify(sentinel),
483
+
},
484
+
{ benchmark: { enabled: true }, provide: { options: fastBenchOptions } },
485
+
)
486
+
expect(stderr).toBe('')
487
+
const after = JSON.parse(fs.readFile('out/upd.json'))
488
+
expect(after.latency.mean).not.toBe(99999)
489
+
})
490
+
491
+
// eslint-disable-next-line no-template-curly-in-string
492
+
test('`writeResult` substitutes `${projectName}` so multi-project runs do not collide', async () => {
493
+
const { stderr, fs } = await runInlineTests(
494
+
{
495
+
'shared.bench.ts': /* ts */`
496
+
import { test, inject } from 'vitest'
497
+
test('t', async ({ bench }) => {
498
+
await bench(
499
+
'x',
500
+
{ writeResult: './out/x.\${projectName}.json' },
501
+
() => {},
502
+
).run(inject('options'))
503
+
})
504
+
`,
505
+
},
506
+
{
507
+
projects: [
508
+
{ test: { name: 'one', benchmark: { enabled: true } } },
509
+
{ test: { name: 'two', benchmark: { enabled: true } } },
510
+
],
511
+
provide: { options: fastBenchOptions },
512
+
},
513
+
)
514
+
expect(stderr).toBe('')
515
+
// ${projectName} substitutes to the parent project name (not the cloned
516
+
// bench project name) so users never see the internal " (bench)" suffix
517
+
// bleed into their paths.
518
+
expect(typeof JSON.parse(fs.readFile('out/x.one.json')).latency.mean).toBe('number')
519
+
expect(typeof JSON.parse(fs.readFile('out/x.two.json')).latency.mean).toBe('number')
520
+
})
521
+
522
+
test('`writeResult` does NOT write a file when the benchmark throws', async () => {
523
+
const { fs } = await runInlineTests(
524
+
{
525
+
'throw.bench.ts': /* ts */`
526
+
import { test, inject } from 'vitest'
527
+
test('throws', async ({ bench }) => {
528
+
try {
529
+
await bench(
530
+
'x',
531
+
{ writeResult: './out/should-not-exist.json' },
532
+
() => { throw new Error('boom') },
533
+
).run(inject('options'))
534
+
} catch {}
535
+
})
536
+
`,
537
+
},
538
+
{ benchmark: { enabled: true }, provide: { options: fastBenchOptions } },
539
+
)
540
+
expect(() => fs.readFile('out/should-not-exist.json')).toThrow()
541
+
})
542
+
543
+
test('`bench.from(name, path)` reads a stored result without invoking any function', async () => {
544
+
const seed = fakeBaseline(0.5)
545
+
const { stderr, results } = await runInlineTests(
546
+
{
547
+
'read.bench.ts': /* ts */`
548
+
import { test, expect } from 'vitest'
549
+
test('read', async ({ bench }) => {
550
+
const r = await bench.from('previous', './out/seed.json').run()
551
+
expect(r.latency.mean).toBe(0.5)
552
+
})
553
+
`,
554
+
'out/seed.json': JSON.stringify(seed),
555
+
},
556
+
{ benchmark: { enabled: true } },
557
+
)
558
+
expect(stderr).toBe('')
559
+
expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed')
560
+
})
561
+
562
+
test('`bench.from(name, fn)` awaits the function and treats its return value as the result', async () => {
563
+
await runPassingBench('readfn.bench.ts', /* ts */`
564
+
import { test, expect } from 'vitest'
565
+
test('read via function', async ({ bench }) => {
566
+
const data = {
567
+
latency: { mean: 0.42, min: 0.42, max: 0.42, samplesCount: 1 },
568
+
throughput: { mean: 1, min: 1, max: 1, samplesCount: 1 },
569
+
period: 0.42,
570
+
totalTime: 0.42,
571
+
}
572
+
const r = await bench.from('previous', () => Promise.resolve(data)).run()
573
+
expect(r.latency.mean).toBe(0.42)
574
+
})
575
+
`)
576
+
})
577
+
578
+
test('`bench.from()` raises a helpful error when the file is missing', async () => {
579
+
await runPassingBench('missingfile.bench.ts', /* ts */`
580
+
import { test, expect } from 'vitest'
581
+
test('missing file', async ({ bench }) => {
582
+
await expect(bench.from('x', './does-not-exist.json').run()).rejects.toThrow(
583
+
/could not find a result file at/,
584
+
)
585
+
})
586
+
`)
587
+
})
588
+
589
+
test('`bench.from()` rejects a path that escapes the project root', async () => {
590
+
await runPassingBench('escape.bench.ts', /* ts */`
591
+
import { test, expect } from 'vitest'
592
+
test('escape', async ({ bench }) => {
593
+
await expect(bench.from('x', '../package.json').run()).rejects.toThrow(
594
+
/resolves outside the project root/,
595
+
)
596
+
})
597
+
`)
598
+
})
599
+
600
+
test('`bench.compare` with a live `writeResult` AND a `bench.from()` records both tasks', async () => {
601
+
const seed = fakeBaseline(0.5)
602
+
const tasks: TestBenchmarkTask[] = []
603
+
const { stderr, results } = await runInlineTests(
604
+
{
605
+
'mixed.bench.ts': /* ts */`
606
+
import { test, expect, inject } from 'vitest'
607
+
test('mixed', async ({ bench }) => {
608
+
const storage = await bench.compare(
609
+
bench('current', { writeResult: './out/current.json' }, () => {}),
610
+
bench.from('previous', './out/previous.json'),
611
+
inject('options'),
612
+
)
613
+
expect(storage.get('previous').latency.mean).toBe(0.5)
614
+
expect(typeof storage.get('current').latency.mean).toBe('number')
615
+
})
616
+
`,
617
+
'out/previous.json': JSON.stringify(seed),
618
+
},
619
+
{
620
+
benchmark: { enabled: true },
621
+
reporters: [{
622
+
onTestCaseBenchmark(_tc, benchmark) {
623
+
tasks.push(...benchmark.tasks)
624
+
},
625
+
}],
626
+
provide: { options: fastBenchOptions },
627
+
},
628
+
)
629
+
expect(stderr).toBe('')
630
+
expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed')
631
+
// both rows appear in the same TestBenchmark, with the from() row marked
632
+
expect(tasks.map(t => ({ name: t.name, fromStore: !!t.fromStore })).sort((a, b) => a.name.localeCompare(b.name)))
633
+
.toEqual([
634
+
{ name: 'current', fromStore: false },
635
+
{ name: 'previous', fromStore: true },
636
+
])
637
+
})
638
+
639
+
test('`bench.compare` with only `bench.from()` registrations skips tinybench entirely', async () => {
640
+
const a = fakeBaseline(0.5)
641
+
const b = fakeBaseline(1)
642
+
const tasks: TestBenchmarkTask[] = []
643
+
const { stderr, results } = await runInlineTests(
644
+
{
645
+
'only-from.bench.ts': /* ts */`
646
+
import { test, expect } from 'vitest'
647
+
test('only from', async ({ bench }) => {
648
+
const storage = await bench.compare(
649
+
bench.from('a', './out/a.json'),
650
+
bench.from('b', './out/b.json'),
651
+
)
652
+
expect(storage.get('a').latency.mean).toBe(0.5)
653
+
expect(storage.get('b').latency.mean).toBe(1)
654
+
})
655
+
`,
656
+
'out/a.json': JSON.stringify(a),
657
+
'out/b.json': JSON.stringify(b),
658
+
},
659
+
{
660
+
benchmark: { enabled: true },
661
+
reporters: [{
662
+
onTestCaseBenchmark(_tc, benchmark) {
663
+
tasks.push(...benchmark.tasks)
664
+
},
665
+
}],
666
+
},
667
+
)
668
+
expect(stderr).toBe('')
669
+
expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed')
670
+
expect(tasks.every(t => t.fromStore)).toBe(true)
671
+
})
672
+
673
+
test('`bench.from()` rows render rme and samples columns from the stored data', async () => {
674
+
// The on-disk BaselineData includes the full `latency` Statistics — including
675
+
// `rme` and `samplesCount` — so a stored row should display real numbers in
676
+
// those columns, not placeholder dashes.
677
+
const seed = { ...fakeBaseline(0.5), latency: { ...fakeStats(0.5), rme: 1.23, samplesCount: 7 } }
678
+
const { stderr, stdout } = await runInlineTests(
679
+
{
680
+
'render.bench.ts': /* ts */`
681
+
import { test, inject } from 'vitest'
682
+
test('render', async ({ bench }) => {
683
+
await bench.compare(
684
+
bench('live', () => {}),
685
+
bench.from('stored', './out/from.json'),
686
+
inject('options'),
687
+
)
688
+
})
689
+
`,
690
+
'out/from.json': JSON.stringify(seed),
691
+
},
692
+
{
693
+
benchmark: { enabled: true },
694
+
reporters: ['default'],
695
+
provide: { options: fastBenchOptions },
696
+
},
697
+
)
698
+
expect(stderr).toBe('')
699
+
const lines = stdout.split('\n')
700
+
const headerIdx = lines.findIndex(l => /^\s*name\s+hz\s+min/.test(l))
701
+
expect(headerIdx, `inline table header not found in stdout:\n${stdout}`).toBeGreaterThanOrEqual(0)
702
+
// The header columns are: name, hz, min, max, mean, p75, p99, p995, p999, rme, samples.
703
+
// Find the row for "stored" and inspect its last two cells.
704
+
const storedRow = lines.slice(headerIdx + 1, headerIdx + 3).find(l => /^\s*stored\b/.test(l))!
705
+
expect(storedRow, `stored row not found in:\n${stdout}`).toBeDefined()
706
+
const cells = storedRow.trim().replace(/\s+(?:fastest|slowest)\s*$/, '').split(/\s+/)
707
+
// rme + samples must be real values, not the `-` placeholder
708
+
expect(cells[cells.length - 2]).toBe('±1.23%')
709
+
expect(cells[cells.length - 1]).toBe('7')
710
+
})
711
+
712
+
test('`benchmark.include` overrides the default `*.bench.ts` pattern', async () => {
713
+
const tasks: TestBenchmarkTask[] = []
714
+
const { stderr } = await runInlineTests(
715
+
{
716
+
'a.perf.ts': /* ts */`
717
+
import { test, inject } from 'vitest'
718
+
test('custom include', async ({ bench }) => {
719
+
await bench('x', () => {}).run(inject('options'))
720
+
})
721
+
`,
722
+
// this .bench.ts would run under the default pattern — but we override
723
+
'b.bench.ts': /* ts */`
724
+
import { test } from 'vitest'
725
+
test('should not run', async ({ bench }) => {
726
+
await bench('never', () => { throw new Error('not discovered') }).run()
727
+
})
728
+
`,
729
+
},
730
+
{
731
+
benchmark: { enabled: true, include: ['**/*.perf.ts'] },
732
+
reporters: [{
733
+
onTestCaseBenchmark(_tc, benchmark) {
734
+
tasks.push(...benchmark.tasks)
735
+
},
736
+
}],
737
+
provide: { options: fastBenchOptions },
738
+
},
739
+
)
740
+
expect(stderr).toBe('')
741
+
expect(tasks.map(t => t.name)).toEqual(['x'])
742
+
})
743
+
744
+
test('`benchmark.exclude` filters matching files out of the bench project', async () => {
745
+
const tasks: TestBenchmarkTask[] = []
746
+
const { stderr } = await runInlineTests(
747
+
{
748
+
'wanted.bench.ts': /* ts */`
749
+
import { test, inject } from 'vitest'
750
+
test('wanted', async ({ bench }) => {
751
+
await bench('x', () => {}).run(inject('options'))
752
+
})
753
+
`,
754
+
'skipped.bench.ts': /* ts */`
755
+
import { test } from 'vitest'
756
+
test('should not run', async ({ bench }) => {
757
+
await bench('never', () => { throw new Error('excluded') }).run()
758
+
})
759
+
`,
760
+
},
761
+
{
762
+
benchmark: {
763
+
enabled: true,
764
+
exclude: ['**/skipped.bench.ts', '**/node_modules/**'],
765
+
},
766
+
reporters: [{
767
+
onTestCaseBenchmark(_tc, benchmark) {
768
+
tasks.push(...benchmark.tasks)
769
+
},
770
+
}],
771
+
provide: { options: fastBenchOptions },
772
+
},
773
+
)
774
+
expect(stderr).toBe('')
775
+
expect(tasks.map(t => t.name)).toEqual(['x'])
776
+
})
777
+
778
+
test('`benchmark.includeSource` runs in-source benchmarks via `import.meta.vitest`', async () => {
779
+
const tasks: TestBenchmarkTask[] = []
780
+
const { stderr } = await runInlineTests(
781
+
{
782
+
'lib.ts': /* ts */`
783
+
export function add(a: number, b: number) { return a + b }
784
+
if (import.meta.vitest) {
785
+
const { test } = import.meta.vitest
786
+
test('in-source', async ({ bench }) => {
787
+
await bench('add', () => add(1, 2)).run({
788
+
time: 0, iterations: 2, warmupTime: 0, warmupIterations: 0,
789
+
})
790
+
})
791
+
}
792
+
`,
793
+
},
794
+
{
795
+
benchmark: { enabled: true, includeSource: ['**/*.ts'] },
796
+
reporters: [{
797
+
onTestCaseBenchmark(_tc, benchmark) {
798
+
tasks.push(...benchmark.tasks)
799
+
},
800
+
}],
801
+
},
802
+
)
803
+
expect(stderr).toBe('')
804
+
expect(tasks.map(t => t.name)).toEqual(['add'])
805
+
})
806
+
807
+
test('`benchmark.retainSamples: true` preserves the raw samples array', async () => {
808
+
const tasks: TestBenchmarkTask[] = []
809
+
const { stderr } = await runInlineTests(
810
+
{
811
+
'samples.bench.ts': /* ts */`
812
+
import { test, inject } from 'vitest'
813
+
test('samples', async ({ bench }) => {
814
+
await bench('x', () => {}).run(inject('options'))
815
+
})
816
+
`,
817
+
},
818
+
{
819
+
benchmark: { enabled: true, retainSamples: true },
820
+
reporters: [{
821
+
onTestCaseBenchmark(_tc, benchmark) {
822
+
tasks.push(...benchmark.tasks)
823
+
},
824
+
}],
825
+
provide: { options: fastBenchOptions },
826
+
},
827
+
)
828
+
expect(stderr).toBe('')
829
+
expect(tasks).toHaveLength(1)
830
+
const samples = tasks[0].latency.samples
831
+
expect(Array.isArray(samples)).toBe(true)
832
+
expect(samples!.length).toBeGreaterThanOrEqual(1)
833
+
})
834
+
835
+
test('`benchmark.retainSamples: false` (the default) omits the samples array', async () => {
836
+
const tasks: TestBenchmarkTask[] = []
837
+
const { stderr } = await runInlineTests(
838
+
{
839
+
'nosamples.bench.ts': /* ts */`
840
+
import { test, inject } from 'vitest'
841
+
test('nosamples', async ({ bench }) => {
842
+
await bench('x', () => {}).run(inject('options'))
843
+
})
844
+
`,
845
+
},
846
+
{
847
+
benchmark: { enabled: true },
848
+
reporters: [{
849
+
onTestCaseBenchmark(_tc, benchmark) {
850
+
tasks.push(...benchmark.tasks)
851
+
},
852
+
}],
853
+
provide: { options: fastBenchOptions },
854
+
},
855
+
)
856
+
expect(stderr).toBe('')
857
+
expect(tasks).toHaveLength(1)
858
+
expect(tasks[0].latency.samples).toBeUndefined()
859
+
})
860
+
861
+
test('`benchmark.enabled: false` skips .bench.ts files entirely', async () => {
862
+
const names: string[] = []
863
+
const { stderr } = await runInlineTests(
864
+
{
865
+
'ignored.bench.ts': /* ts */`
866
+
import { test } from 'vitest'
867
+
test('should-never-run', () => {
868
+
throw new Error('bench project should not have been created')
869
+
})
870
+
`,
871
+
'regular.test.ts': /* ts */`
872
+
import { test } from 'vitest'
873
+
test('runs', () => {})
874
+
`,
875
+
},
876
+
{
877
+
// benchmark.enabled defaults to false → no bench project cloned
878
+
reporters: [{
879
+
onTestCaseReady(testCase) {
880
+
names.push(testCase.name)
881
+
},
882
+
}],
883
+
},
884
+
)
885
+
expect(stderr).toBe('')
886
+
expect(names).toEqual(['runs'])
887
+
})
888
+
889
+
test('`vitest bench` CLI invocation filters to the cloned benchmark project', async () => {
890
+
const { stderr, ctx } = await runInlineTests(
891
+
{
892
+
'x.bench.ts': /* ts */`
893
+
import { test, inject } from 'vitest'
894
+
test('x', async ({ bench }) => {
895
+
await bench('a', () => {}).run(inject('options'))
896
+
})
897
+
`,
898
+
},
899
+
{
900
+
$cliOptions: { benchmarkOnly: true },
901
+
provide: { options: fastBenchOptions },
902
+
},
903
+
)
904
+
expect(stderr).toBe('')
905
+
// --benchmarkOnly narrows ctx.projects to only bench-enabled projects
906
+
const projectNames = ctx?.projects.map(p => p.name) ?? []
907
+
expect(projectNames).toEqual(['bench'])
908
+
})
909
+
910
+
test('json reporter surfaces benchmarks on each assertion result', async () => {
911
+
const { stderr, stdout } = await runInlineTests(
912
+
{
913
+
'foo.bench.ts': /* ts */`
914
+
import { test, inject } from 'vitest'
915
+
test('smoke', async ({ bench }) => {
916
+
await bench.compare(
917
+
bench('a', () => {}),
918
+
bench('b', () => {}),
919
+
inject('options'),
920
+
)
921
+
})
922
+
`,
923
+
},
924
+
{
925
+
benchmark: { enabled: true },
926
+
reporters: 'json',
927
+
provide: { options: fastBenchOptions },
928
+
},
929
+
)
930
+
expect(stderr).toBe('')
931
+
const parsed = JSON.parse(stdout) as JsonTestResults
932
+
const assertionResults = parsed.testResults.flatMap(tr => tr.assertionResults)
933
+
const smoke = assertionResults.find(a => a.title === 'smoke')!
934
+
expect(
935
+
smoke,
936
+
`no assertion result titled "smoke" in json output:\n${JSON.stringify(parsed, null, 2)}`,
937
+
).toBeDefined()
938
+
expect(Array.isArray(smoke.benchmarks)).toBe(true)
939
+
expect(smoke.benchmarks).toHaveLength(1)
940
+
const [benchmark] = smoke.benchmarks
941
+
expect(benchmark.tasks.map(t => t.name).sort()).toEqual(['a', 'b'])
942
+
// tasks are ranked 1..n and carry the full statistics surface
943
+
expect(benchmark.tasks.map(t => t.rank).sort()).toEqual([1, 2])
944
+
expect(typeof benchmark.tasks[0].latency.mean).toBe('number')
945
+
expect(typeof benchmark.tasks[0].throughput.mean).toBe('number')
946
+
})
947
+
948
+
test('multi-project run aggregates perProject tasks into a single cross-project sub-table', async () => {
949
+
const { stderr, stdout } = await runInlineTests(
950
+
{
951
+
'shared.bench.ts': /* ts */`
952
+
import { test, inject } from 'vitest'
953
+
test('cross', async ({ bench }) => {
954
+
await bench('x', { perProject: true }, () => {}).run(inject('options'))
955
+
})
956
+
`,
957
+
},
958
+
{
959
+
projects: [
960
+
{ test: { name: 'one', benchmark: { enabled: true } } },
961
+
{ test: { name: 'two', benchmark: { enabled: true } } },
962
+
],
963
+
reporters: ['default'],
964
+
provide: { options: fastBenchOptions },
965
+
},
966
+
)
967
+
expect(stderr).toBe('')
968
+
expect(stdout).toContain('Cross-Project Benchmark Comparison')
969
+
970
+
// find the `project ... hz ... min` header of the `x` sub-table + 2 data rows
971
+
const lines = stdout.split('\n')
972
+
const headerIdx = lines.findIndex(l => /^\s*project\s+hz\s+min/.test(l))
973
+
expect(headerIdx, `cross-project sub-table header not found in stdout:\n${stdout}`).toBeGreaterThan(-1)
974
+
const [header, ...rows] = lines.slice(headerIdx, headerIdx + 3)
975
+
const normalized = formatBenchTable([
976
+
header,
977
+
...rows.map(r => r.replace(/\s+(?:fastest|slowest)\s*$/, '')).sort(),
978
+
])
979
+
expect(normalized).toMatchInlineSnapshot(`
980
+
" project hz min max mean p75 p99 p995 p999 rme samples
981
+
one (bench) d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+
982
+
two (bench) d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+"
983
+
`)
984
+
})
985
+
986
+
test('cross-project section is skipped when every perProject benchmark ran in only one project', async () => {
987
+
// Even when several perProject benchmarks are recorded, the cross-project
988
+
// table is useless if each one ran in exactly one project — every sub-table
989
+
// would be a single row with nothing to compare against.
990
+
const { stderr, stdout } = await runInlineTests(
991
+
{
992
+
'solo.bench.ts': /* ts */`
993
+
import { test, inject } from 'vitest'
994
+
test('solo', async ({ bench }) => {
995
+
await bench.compare(
996
+
bench('a', { perProject: true }, () => {}),
997
+
bench('b', { perProject: true }, () => {}),
998
+
inject('options'),
999
+
)
1000
+
})
1001
+
`,
1002
+
},
1003
+
{
1004
+
benchmark: { enabled: true },
1005
+
reporters: ['default'],
1006
+
provide: { options: fastBenchOptions },
1007
+
},
1008
+
)
1009
+
expect(stderr).toBe('')
1010
+
expect(stdout).not.toContain('Cross-Project Benchmark Comparison')
1011
+
})
1012
+
1013
+
test('cross-project section is absent when no benchmark is perProject', async () => {
1014
+
const { stderr, stdout } = await runInlineTests(
1015
+
{
1016
+
'nox.bench.ts': /* ts */`
1017
+
import { test, inject } from 'vitest'
1018
+
test('no perProject', async ({ bench }) => {
1019
+
await bench('only', () => {}).run(inject('options'))
1020
+
})
1021
+
`,
1022
+
},
1023
+
{
1024
+
benchmark: { enabled: true },
1025
+
reporters: ['default'],
1026
+
provide: { options: fastBenchOptions },
1027
+
},
1028
+
)
1029
+
expect(stderr).toBe('')
1030
+
expect(stdout).not.toContain('Cross-Project Benchmark Comparison')
1031
+
})
1032
+
1033
+
test('`bench.compare` wraps multiple failed benchmarks in an AggregateError', async () => {
1034
+
await runPassingBench('aggregate.bench.ts', /* ts */`
1035
+
import { test, expect, inject } from 'vitest'
1036
+
test('aggregate errors', async ({ bench }) => {
1037
+
const err = await bench.compare(
1038
+
bench('a', () => { throw new Error('A failed') }),
1039
+
bench('b', () => { throw new Error('B failed') }),
1040
+
inject('options'),
1041
+
).catch(e => e)
1042
+
expect(err).toBeInstanceOf(AggregateError)
1043
+
expect(err.message).toBe('Some benchmarks failed')
1044
+
const messages = err.errors.map((e) => e.message).sort()
1045
+
expect(messages).toEqual(['A failed', 'B failed'])
1046
+
})
1047
+
`)
1048
+
})
1049
+
1050
+
test('`BenchStorage.get` returns a valid BenchResult shape for every registration', async () => {
1051
+
await runPassingBench('storage.bench.ts', /* ts */`
1052
+
import { test, expect, inject } from 'vitest'
1053
+
test('storage shape', async ({ bench }) => {
1054
+
const storage = await bench.compare(
1055
+
bench('a', () => {}),
1056
+
bench('b', () => {}),
1057
+
inject('options'),
1058
+
)
1059
+
for (const name of ['a', 'b'] as const) {
1060
+
const result = storage.get(name)
1061
+
expect(typeof result.latency.mean).toBe('number')
1062
+
expect(typeof result.throughput.mean).toBe('number')
1063
+
expect(typeof result.period).toBe('number')
1064
+
expect(typeof result.totalTime).toBe('number')
1065
+
}
1066
+
})
1067
+
`)
1068
+
})
1069
+
1070
+
test('`bench.compare` trailing options propagate through to the underlying Tinybench', async () => {
1071
+
const benchmarks: TestBenchmark[] = []
1072
+
const { stderr } = await runInlineTests(
1073
+
{
1074
+
'options.bench.ts': /* ts */`
1075
+
import { test, inject } from 'vitest'
1076
+
test('options', async ({ bench }) => {
1077
+
await bench.compare(
1078
+
bench('a', () => {}),
1079
+
bench('b', () => {}),
1080
+
{ ...inject('options'), name: 'custom-bench-name' },
1081
+
)
1082
+
})
1083
+
`,
1084
+
},
1085
+
{
1086
+
benchmark: { enabled: true },
1087
+
reporters: [{
1088
+
onTestCaseBenchmark(_tc, benchmark) {
1089
+
benchmarks.push(benchmark)
1090
+
},
1091
+
}],
1092
+
provide: { options: fastBenchOptions },
1093
+
},
1094
+
)
1095
+
expect(stderr).toBe('')
1096
+
expect(benchmarks).toHaveLength(1)
1097
+
// user-supplied `name` overrides the default `<fullTestName> <idx>` label;
1098
+
// the serialized benchmark emitted to reporters carries it verbatim
1099
+
expect(benchmarks[0].name).toBe('custom-bench-name')
1100
+
})
1101
+
1102
+
test('benchmark warns when module export getters are accessed too many times', async () => {
1103
+
const { stderr } = await runInlineTests(
1104
+
{
1105
+
'fixture.ts': /* ts */`
1106
+
export const value = 1
1107
+
`,
1108
+
'getter-warning.bench.ts': /* ts */`
1109
+
import { test, inject } from 'vitest'
1110
+
import * as fixture from './fixture'
1111
+
1112
+
test('getter warning', async ({ bench }) => {
1113
+
await bench('read getter', () => {
1114
+
for (let i = 0; i < 1_000_001; i++) {
1115
+
void fixture.value
1116
+
}
1117
+
}).run(inject('options'))
1118
+
})
1119
+
`,
1120
+
},
1121
+
{
1122
+
benchmark: { enabled: true },
1123
+
provide: { options: { ...fastBenchOptions, iterations: 1 } },
1124
+
},
1125
+
)
1126
+
1127
+
expect(stderr).toMatchInlineSnapshot(`
1128
+
"stderr | getter-warning.bench.ts > getter warning
1129
+
Benchmark Warning
1130
+
Benchmark "getter warning 1" accessed module export getters too many times.
1131
+
1132
+
This can make results unreliable because export getters add overhead.
1133
+
See https://vitest.dev/guide/benchmarking#module-runner-overhead
1134
+
1135
+
Tracked exports:
1136
+
- fixture.ts > value
1137
+
1138
+
"
1139
+
`)
1140
+
})
1141
+
1142
+
test('benchmark export getter warning can be suppressed', async () => {
1143
+
const { stderr } = await runInlineTests(
1144
+
{
1145
+
'fixture.ts': /* ts */`
1146
+
export const value = 1
1147
+
`,
1148
+
'getter-warning.bench.ts': /* ts */`
1149
+
import { test, inject } from 'vitest'
1150
+
import * as fixture from './fixture'
1151
+
1152
+
test('getter warning', async ({ bench }) => {
1153
+
await bench('read getter', () => {
1154
+
for (let i = 0; i < 1_000_001; i++) {
1155
+
void fixture.value
1156
+
}
1157
+
}).run(inject('options'))
1158
+
})
1159
+
`,
1160
+
},
1161
+
{
1162
+
benchmark: { enabled: true, suppressExportGetterWarnings: true },
1163
+
provide: { options: { ...fastBenchOptions, iterations: 1 } },
1164
+
},
1165
+
)
1166
+
1167
+
expect(stderr).toBe('')
1168
+
})
1169
+
1170
+
test('warns when `bench()` is registered but never run', async () => {
1171
+
const { stderr } = await runInlineTests(
1172
+
{
1173
+
'unrun.bench.ts': /* ts */`
1174
+
import { test } from 'vitest'
1175
+
test('forgot to run', ({ bench }) => {
1176
+
bench('a', () => {})
1177
+
})
1178
+
`,
1179
+
},
1180
+
{ benchmark: { enabled: true } },
1181
+
)
1182
+
expect(stderr).toContain('Benchmark Warning')
1183
+
expect(stderr).toContain('forgot to run')
1184
+
expect(stderr).toContain('"a"')
1185
+
})
1186
+
1187
+
test('warns about every unrun registration in the message, including `bench.from()`', async () => {
1188
+
const { stderr } = await runInlineTests(
1189
+
{
1190
+
'out/seed.json': JSON.stringify(fakeBaseline(0.5)),
1191
+
'multi.bench.ts': /* ts */`
1192
+
import { test } from 'vitest'
1193
+
test('multi unrun', ({ bench }) => {
1194
+
bench('a', () => {})
1195
+
bench('b', () => {})
1196
+
bench.from('seed', './out/seed.json')
1197
+
})
1198
+
`,
1199
+
},
1200
+
{ benchmark: { enabled: true } },
1201
+
)
1202
+
expect(stderr).toContain('Benchmark Warning')
1203
+
// every name appears in a single warning line
1204
+
expect(stderr).toMatch(/"a".+"b".+"seed"/)
1205
+
})
1206
+
1207
+
test('does NOT warn when every registration is consumed by `.run()` or `bench.compare()`', async () => {
1208
+
const { stderr } = await runInlineTests(
1209
+
{
1210
+
'consumed.bench.ts': /* ts */`
1211
+
import { test, inject } from 'vitest'
1212
+
test('all consumed', async ({ bench }) => {
1213
+
await bench('lone', () => {}).run(inject('options'))
1214
+
await bench.compare(
1215
+
bench('x', () => {}),
1216
+
bench('y', () => {}),
1217
+
inject('options'),
1218
+
)
1219
+
})
1220
+
`,
1221
+
},
1222
+
{ benchmark: { enabled: true }, provide: { options: fastBenchOptions } },
1223
+
)
1224
+
expect(stderr).toBe('')
1225
+
})
1226
+
1227
+
test('warns only about the unrun registration when others are consumed', async () => {
1228
+
const { stderr } = await runInlineTests(
1229
+
{
1230
+
'partial.bench.ts': /* ts */`
1231
+
import { test, inject } from 'vitest'
1232
+
test('partial', async ({ bench }) => {
1233
+
await bench('used', () => {}).run(inject('options'))
1234
+
bench('forgotten', () => {})
1235
+
})
1236
+
`,
1237
+
},
1238
+
{ benchmark: { enabled: true }, provide: { options: fastBenchOptions } },
1239
+
)
1240
+
expect(stderr).toContain('Benchmark Warning')
1241
+
expect(stderr).toContain('"forgotten"')
1242
+
expect(stderr).not.toContain('"used"')
1243
+
})
1244
+
1245
+
test('`BenchStorage.get("missing")` throws a descriptive error', async () => {
1246
+
await runPassingBench('missing.bench.ts', /* ts */`
1247
+
import { test, expect, inject } from 'vitest'
1248
+
test('missing', async ({ bench }) => {
1249
+
const storage = await bench.compare(
1250
+
bench('a', () => {}),
1251
+
bench('b', () => {}),
1252
+
inject('options'),
1253
+
)
1254
+
expect(() => storage.get('missing')).toThrow(
1255
+
/task "missing" was not defined/,
1256
+
)
1257
+
})
1258
+
`)
1259
+
})
1260
+
1261
+
test('`toBeFasterThan` passes when actual.latency.mean is strictly smaller', () => {
1262
+
expect(fakeResult(0.5)).toBeFasterThan(fakeResult(1.0))
1263
+
})
1264
+
1265
+
test('`toBeFasterThan` fails with a percent-slower message when actual is slower', () => {
1266
+
expect(() => expect(fakeResult(1.0)).toBeFasterThan(fakeResult(0.5)))
1267
+
.toThrowErrorMatchingInlineSnapshot(`
1268
+
[Error: [2mexpect([22m[31mreceived[39m[2m).toBeFasterThan([22m[32mexpected[39m[2m)[22m
1269
+
1270
+
Expected to be faster, but was 100.00% slower.
1271
+
1272
+
Received: [31m1.00[39m ops/sec
1273
+
Expected: [32m2.00[39m ops/sec
1274
+
]
1275
+
`)
1276
+
})
1277
+
1278
+
test('`toBeFasterThan` honours the `delta` threshold', () => {
1279
+
const fast = fakeResult(0.8) // 20% faster than 1.0
1280
+
const slow = fakeResult(1.0)
1281
+
// 20% faster is not enough when delta demands 30%
1282
+
expect(() => expect(fast).toBeFasterThan(slow, { delta: 0.3 }))
1283
+
.toThrow(/faster by at least 30%/)
1284
+
// passes when the demanded margin is only 10%
1285
+
expect(fast).toBeFasterThan(slow, { delta: 0.1 })
1286
+
})
1287
+
1288
+
test('`toBeSlowerThan` passes when actual.latency.mean is strictly larger', () => {
1289
+
expect(fakeResult(1.0)).toBeSlowerThan(fakeResult(0.5))
1290
+
})
1291
+
1292
+
test('`toBeSlowerThan` fails with a percent-faster message when actual is faster', () => {
1293
+
expect(() => expect(fakeResult(0.5)).toBeSlowerThan(fakeResult(1.0)))
1294
+
.toThrowErrorMatchingInlineSnapshot(`
1295
+
[Error: [2mexpect([22m[31mreceived[39m[2m).toBeSlowerThan([22m[32mexpected[39m[2m)[22m
1296
+
1297
+
Expected to be slower, but was 50% faster.
1298
+
1299
+
Received: [31m2.00[39m ops/sec
1300
+
Expected: [32m1.00[39m ops/sec
1301
+
]
1302
+
`)
1303
+
})
1304
+
1305
+
test('`toBeSlowerThan` honours the `delta` threshold', () => {
1306
+
// slow = 2x fast → 100% slower. delta 0.5 → threshold 150% → passes at 100%
1307
+
expect(fakeResult(1.0)).toBeSlowerThan(fakeResult(0.5), { delta: 0.5 })
1308
+
expect(() => expect(fakeResult(1.0)).toBeSlowerThan(fakeResult(0.5), { delta: 1.5 }))
1309
+
.toThrow(/slower by at least 150%/)
1310
+
})
1311
+
1312
+
test('bench matchers reject non-benchmark-result values with a TypeError', () => {
1313
+
expect(() => expect({ foo: 'bar' }).toBeFasterThan(fakeResult(1.0)))
1314
+
.toThrow(TypeError)
1315
+
expect(() => expect(fakeResult(1.0)).toBeFasterThan({ foo: 'bar' } as any))
1316
+
.toThrow(TypeError)
1317
+
expect(() => expect({ foo: 'bar' }).toBeSlowerThan(fakeResult(1.0)))
1318
+
.toThrow(TypeError)
1319
+
})
1320
+
1321
+
declare module 'vitest' {
1322
+
interface ProvidedContext {
1323
+
options: typeof fastBenchOptions
1324
+
}
1325
+
}
-45
test/e2e/test/mode.test.ts
-45
test/e2e/test/mode.test.ts
···
1
-
import { expect, test } from 'vitest'
2
-
import * as testUtils from '../../test-utils'
3
-
4
-
test.each([
5
-
{ expectedMode: 'test', command: ['run'] },
6
-
{ expectedMode: 'benchmark', command: ['bench', '--run'] },
7
-
])(`env.mode should have the $expectedMode value when running in $name mode`, async ({ command, expectedMode }) => {
8
-
const { stdout } = await testUtils.runVitestCli(...(command), 'fixtures/mode', '-c', `fixtures/mode/vitest.${expectedMode}.config.ts`)
9
-
10
-
expect(stdout).toContain(`✓ fixtures/mode/example.${expectedMode}.ts`)
11
-
})
12
-
13
-
test.each([
14
-
{ expectedMode: 'test', command: ['bench', '--run'], actualMode: 'benchmark' },
15
-
{ expectedMode: 'benchmark', command: ['run'], actualMode: 'test' },
16
-
])(`should return error if actual mode $actualMode is different than expected mode $expectedMode`, async ({ command, expectedMode, actualMode }) => {
17
-
const { stdout, stderr } = await testUtils.runVitestCli(...(command), 'fixtures/mode', '-c', `fixtures/mode/vitest.${expectedMode}.config.ts`)
18
-
19
-
expect(stderr).toContain(`env.mode: ${actualMode}`)
20
-
expect(stderr).toContain('Startup Error')
21
-
expect(stderr).toContain(`Error: env.mode should be equal to "${expectedMode}"`)
22
-
expect(stdout).toBe('')
23
-
})
24
-
25
-
test.each([
26
-
{ options: ['run'], expected: 'run' },
27
-
{ options: ['run', '--watch'], expected: 'watch' },
28
-
{ options: ['watch'], expected: 'watch' },
29
-
] as const)(`vitest $options.0 $options.1 resolves to $expected-mode`, async ({ options, expected }) => {
30
-
const { vitest } = await testUtils.runVitestCli(...options, '--root', 'fixtures/run-mode')
31
-
32
-
if (expected === 'watch') {
33
-
await vitest.waitForStdout('Test Files 1 passed (1)')
34
-
35
-
expect(vitest.stdout).not.toContain('RUN')
36
-
expect(vitest.stdout).toContain('DEV')
37
-
expect(vitest.stdout).toContain('Waiting for file changes')
38
-
}
39
-
40
-
if (expected === 'run') {
41
-
expect(vitest.stdout).toContain('RUN')
42
-
expect(vitest.stdout).not.toContain('DEV')
43
-
expect(vitest.stdout).not.toContain('Waiting for file changes')
44
-
}
45
-
})
+1
-1
test/e2e/test/public.test.ts
+1
-1
test/e2e/test/public.test.ts
···
20
20
setupFiles: ['/test/setup.ts'],
21
21
})
22
22
expect(viteConfig.mode).toBe('development')
23
-
expect(vitestConfig.mode).toBe('test') // vitest mode is "test" or "benchmark"
23
+
expect(vitestConfig.mode).toBe('development')
24
24
expect(vitestConfig.setupFiles).toEqual(['/test/setup.ts'])
25
25
expect(viteConfig.plugins.find(p => p.name === 'vitest')).toBeDefined()
26
26
})
+1
-1
test/node-runner/test/cli.test.js
+1
-1
test/node-runner/test/cli.test.js
···
2
2
import { startVitest } from 'vitest/node'
3
3
4
4
await test('importing vitest in the global setup is reported as an error', async (t) => {
5
-
const vitest = await startVitest('test', [], {
5
+
const vitest = await startVitest([], {
6
6
root: './fixtures/globalSetup',
7
7
globalSetup: [
8
8
'./failing.ts',
+2
-2
test/ui/test/helper.ts
+2
-2
test/ui/test/helper.ts
···
12
12
export async function startVitestSimple(cliOptions: CliOptions): Promise<Vitest> {
13
13
const stdout = new Writable({ write: (_, __, callback) => callback() })
14
14
const stderr = new Writable({ write: (_, __, callback) => callback() })
15
-
const vitest = await startVitest('test', undefined, cliOptions, {}, { stdout, stderr })
15
+
const vitest = await startVitest(undefined, cliOptions, {}, { stdout, stderr })
16
16
await vitest.close()
17
17
return vitest
18
18
}
···
24
24
// silence Vitest logs
25
25
const stdout = new Writable({ write: (_, __, callback) => callback() })
26
26
const stderr = new Writable({ write: (_, __, callback) => callback() })
27
-
const vitest = await startVitest('test', undefined, cliOptions, viteOverrides, { stdout, stderr })
27
+
const vitest = await startVitest(undefined, cliOptions, viteOverrides, { stdout, stderr })
28
28
29
29
const address = vitest.vite.httpServer?.address()
30
30
assert(address && typeof address === 'object', 'Invalid server address')
-18
test/unit/test/cli-test.test.ts
-18
test/unit/test/cli-test.test.ts
···
133
133
expect(getCLIOptions('--coverage.thresholds.autoUpdate no').coverage.thresholds.autoUpdate).toBe(false)
134
134
})
135
135
136
-
test('bench only options', async () => {
137
-
expect(() =>
138
-
parseArguments('--compare file.json').matchedCommand?.checkUnknownOptions(),
139
-
).toThrowErrorMatchingInlineSnapshot(
140
-
`[CACError: Unknown option \`--compare\`]`,
141
-
)
142
-
143
-
expect(() =>
144
-
parseArguments(
145
-
'bench --compare file.json',
146
-
).matchedCommand?.checkUnknownOptions(),
147
-
).not.toThrow()
148
-
149
-
expect(parseArguments('bench --compare file.json').options).toEqual({
150
-
compare: 'file.json',
151
-
})
152
-
})
153
-
154
136
test('even if coverage is boolean, don\'t fail', () => {
155
137
expect(getCLIOptions('--coverage --coverage.provider v8').coverage).toEqual({
156
138
enabled: true,
-5
test/unit/test/exports.test.ts
-5
test/unit/test/exports.test.ts
···
17
17
expect(manifest.exports).toMatchInlineSnapshot(`
18
18
{
19
19
".": {
20
-
"BenchmarkRunner": "function",
21
20
"EvaluatedModules": "function",
22
21
"Snapshots": "object",
23
22
"TestRunner": "function",
···
29
28
"assertType": "function",
30
29
"beforeAll": "function",
31
30
"beforeEach": "function",
32
-
"bench": "function",
33
31
"chai": "object",
34
32
"createExpect": "function",
35
33
"describe": "function",
···
84
82
"AgentReporter": "function",
85
83
"BaseCoverageProvider": "function",
86
84
"BaseSequencer": "function",
87
-
"BenchmarkReporter": "function",
88
-
"BenchmarkReportsMap": "object",
89
85
"DefaultReporter": "function",
90
86
"DotReporter": "function",
91
87
"ForksPoolWorker": "function",
···
101
97
"TestsNotFoundError": "function",
102
98
"ThreadsPoolWorker": "function",
103
99
"TypecheckPoolWorker": "function",
104
-
"VerboseBenchmarkReporter": "function",
105
100
"VerboseReporter": "function",
106
101
"VitestPackageInstaller": "function",
107
102
"VitestPlugin": "function",
+1
test/unit/vitest-environment-custom/index.ts
+1
test/unit/vitest-environment-custom/index.ts
+22
-2
packages/browser/src/node/plugin.ts
+22
-2
packages/browser/src/node/plugin.ts
···
168
168
// this plugin can be used in different projects, but all of them
169
169
// have the same `include` pattern, so it doesn't matter which project we use
170
170
const project = parentServer.project
171
-
const { testFiles: browserTestFiles } = await project.globTestFiles()
171
+
// only glob benchmarks when a browser-enabled bench project exists in
172
+
// the workspace — keeps the optimizeDeps entries symmetrical across
173
+
// the test/bench project clones so the optimizer doesn't re-scan when
174
+
// the user switches modes
175
+
const hasBrowserBenchProject = parentServer.vitest.projects.some(p =>
176
+
p.config.browser.enabled && p.config.benchmark.enabled,
177
+
)
178
+
const benchInclude = hasBrowserBenchProject
179
+
? project.config.benchmark.include
180
+
: []
181
+
const dir = project.config.dir || project.config.root
182
+
const [{ testFiles: browserTestFiles }, browserBenchFiles] = await Promise.all([
183
+
project.globTestFiles(),
184
+
benchInclude.length > 0
185
+
? project.globFiles(
186
+
benchInclude,
187
+
project.config.benchmark.exclude ?? project.config.exclude,
188
+
dir,
189
+
)
190
+
: [],
191
+
])
172
192
const setupFiles = toArray(project.config.setupFiles)
173
193
174
194
// replace env values - cannot be reassign at runtime
···
179
199
}
180
200
181
201
const entries: string[] = [
182
-
...browserTestFiles,
202
+
...new Set([...browserTestFiles, ...browserBenchFiles]),
183
203
...setupFiles,
184
204
resolve(vitestDist, 'index.js'),
185
205
resolve(vitestDist, 'browser.js'),
+17
packages/browser/src/node/rpc.ts
+17
packages/browser/src/node/rpc.ts
···
202
202
203
203
return vitest._testRun.recordArtifact(id, artifact)
204
204
},
205
+
async onTestBenchmark(testId, benchmark) {
206
+
return vitest._testRun.recordBenchmark(testId, benchmark)
207
+
},
208
+
async readBenchmarkResult(relativePath) {
209
+
checkFileAccess(project.benchmark.resolve(relativePath))
210
+
return project.benchmark.readResult(relativePath)
211
+
},
212
+
async writeBenchmarkResult(relativePath, data) {
213
+
if (!canWrite(project)) {
214
+
vitest.logger.error(
215
+
`[vitest] Cannot write benchmark artifact "${relativePath}" because file writing is disabled. See https://vitest.dev/config/browser/api.`,
216
+
)
217
+
return
218
+
}
219
+
checkFileAccess(project.benchmark.resolve(relativePath))
220
+
return project.benchmark.writeResult(relativePath, data)
221
+
},
205
222
async onTaskUpdate(method, packs, events) {
206
223
if (method === 'collect') {
207
224
vitest.state.updateTasks(packs)
+13
-4
packages/runner/src/types/runner.ts
+13
-4
packages/runner/src/types/runner.ts
···
42
42
hookTimeout: number
43
43
retry: SerializableRetry
44
44
includeTaskLocation: boolean | undefined
45
-
diffOptions?: DiffOptions
46
45
tags: TestTagDefinition[]
47
46
tagsFilter: string[] | undefined
48
47
strictTags: boolean
48
+
49
+
/**
50
+
* @internal
51
+
*/
52
+
_diffOptions?: DiffOptions
49
53
mergeReportsLabel: string | undefined
50
54
}
51
55
···
93
97
| 'test-failure'
94
98
| (string & Record<string, never>)
95
99
100
+
export interface TestTryOptions {
101
+
retry: number
102
+
repeats: number
103
+
}
104
+
96
105
export interface VitestRunner {
97
106
/**
98
107
* First thing that's getting called before actually collecting and running tests.
···
123
132
*/
124
133
onBeforeTryTask?: (
125
134
test: Test,
126
-
options: { retry: number; repeats: number },
135
+
options: TestTryOptions,
127
136
) => unknown
128
137
/**
129
138
* When the task has finished running, but before cleanup hooks are called
···
138
147
*/
139
148
onAfterTryTask?: (
140
149
test: Test,
141
-
options: { retry: number; repeats: number },
150
+
options: TestTryOptions,
142
151
) => unknown
143
152
/**
144
153
* Called after the retry resolution happened. Unlike `onAfterTryTask`, the test now has a new state.
···
146
155
*/
147
156
onAfterRetryTask?: (
148
157
test: Test,
149
-
options: { retry: number; repeats: number },
158
+
options: TestTryOptions,
150
159
) => unknown
151
160
152
161
/**
+38
packages/runner/src/types/tasks.ts
+38
packages/runner/src/types/tasks.ts
···
1
1
import type { Awaitable, TestError } from '@vitest/utils'
2
+
import type { Statistics } from 'tinybench'
2
3
import type { TestFixtures } from '../fixture'
3
4
import type { afterAll, afterEach, aroundAll, aroundEach, beforeAll, beforeEach } from '../hooks'
4
5
import type { kChainableContext, TypedChainableFunction } from '../utils/chain'
···
340
341
*/
341
342
artifacts: TestArtifact[]
342
343
fullTestName: string
344
+
/**
345
+
* An array of benchmark results generated by `context.bench` function.
346
+
* The benchmark is added only after `bench().run` or `bench.compare` is resolved.
347
+
*
348
+
* @experimental
349
+
*/
350
+
benchmarks: TestBenchmark[]
351
+
}
352
+
353
+
export interface TestBenchmark {
354
+
name: string
355
+
tasks: TestBenchmarkTask[]
356
+
}
357
+
358
+
export type TestBenchmarkStatistics = Statistics
359
+
360
+
export interface BaselineData {
361
+
latency: TestBenchmarkStatistics
362
+
throughput: TestBenchmarkStatistics
363
+
period: number
364
+
totalTime: number
365
+
}
366
+
367
+
export interface TestBenchmarkTask {
368
+
name: string
369
+
latency: TestBenchmarkStatistics
370
+
throughput: TestBenchmarkStatistics
371
+
period: number
372
+
totalTime: number
373
+
rank: number
374
+
perProject?: boolean
375
+
/**
376
+
* `true` when the task was produced by `bench.from()` rather than by
377
+
* executing a function. Reporters can use this to render the row as a
378
+
* static reference (no margin of error, no samples).
379
+
*/
380
+
fromStore?: boolean
343
381
}
344
382
345
383
export type Task = Test | Suite | File
+1
packages/vitest/src/node/ast-collect.ts
+1
packages/vitest/src/node/ast-collect.ts
+42
packages/vitest/src/node/benchmark.ts
+42
packages/vitest/src/node/benchmark.ts
···
1
+
import type { BaselineData } from '@vitest/runner'
2
+
import type { TestProject } from './project'
3
+
import { existsSync } from 'node:fs'
4
+
import { mkdir, readFile, writeFile } from 'node:fs/promises'
5
+
import { dirname, isAbsolute, resolve } from 'pathe'
6
+
7
+
export class BenchmarkManager {
8
+
constructor(private project: TestProject) {}
9
+
10
+
// Resolve a user-supplied path against the project root. Reject paths that
11
+
// escape the project root: `bench.from()` accepts arbitrary input, and we
12
+
// never want a benchmark file to be able to read or clobber files outside
13
+
// the workspace.
14
+
public resolve(relativePath: string): string {
15
+
const root = this.project.config.root
16
+
const absolute = isAbsolute(relativePath)
17
+
? resolve(relativePath)
18
+
: resolve(root, relativePath)
19
+
const rootWithSep = root.endsWith('/') ? root : `${root}/`
20
+
if (absolute !== root && !absolute.startsWith(rootWithSep)) {
21
+
throw new Error(
22
+
`Benchmark artifact path "${relativePath}" resolves outside the project root (${root}). `
23
+
+ `Paths passed to \`writeResult\` and \`bench.from()\` must point inside the project.`,
24
+
)
25
+
}
26
+
return absolute
27
+
}
28
+
29
+
async readResult(relativePath: string): Promise<BaselineData | null> {
30
+
const path = this.resolve(relativePath)
31
+
if (!existsSync(path)) {
32
+
return null
33
+
}
34
+
return JSON.parse(await readFile(path, 'utf-8')) as BaselineData
35
+
}
36
+
37
+
async writeResult(relativePath: string, data: BaselineData): Promise<void> {
38
+
const absolute = this.resolve(relativePath)
39
+
await mkdir(dirname(absolute), { recursive: true })
40
+
await writeFile(absolute, `${JSON.stringify(data, null, 2)}\n`, 'utf-8')
41
+
}
42
+
}
+40
-19
packages/vitest/src/node/core.ts
+40
-19
packages/vitest/src/node/core.ts
···
44
44
import { VitestPackageInstaller } from './packageInstaller'
45
45
import { createPool } from './pool'
46
46
import { TestProject } from './project'
47
-
import { getDefaultTestProject, resolveBrowserProjects, resolveProjects } from './projects/resolveProjects'
47
+
import { getDefaultTestProject, resolveDefaultProjects, resolveProjects } from './projects/resolveProjects'
48
48
import { BlobReporter, readBlobs } from './reporters/blob'
49
49
import { HangingProcessReporter } from './reporters/hanging-process'
50
50
import { createReport } from './reporters/report'
51
-
import { createBenchmarkReporters, createReporters } from './reporters/utils'
51
+
import { createReporters } from './reporters/utils'
52
52
import { VitestResolver } from './resolver'
53
53
import { VitestSpecifications } from './specifications'
54
54
import { StateManager } from './state'
···
77
77
* The logger instance used to log messages. It's recommended to use this logger instead of `console`.
78
78
* It's possible to override stdout and stderr streams when initiating Vitest.
79
79
* @example
80
-
* new Vitest('test', {
80
+
* new Vitest({
81
81
* stdout: new Writable(),
82
82
* })
83
83
*/
···
143
143
private _snapshot?: SnapshotManager
144
144
private _coverageProvider?: CoverageProvider | null | undefined
145
145
146
+
/**
147
+
* @deprecated Do not rely on this property, it's always `test`. Scheduled to be removed in the next major.
148
+
*/
149
+
public readonly mode = 'test'
150
+
146
151
constructor(
147
-
public readonly mode: VitestRunMode,
148
152
cliOptions: UserConfig,
149
-
options: VitestOptions = {},
153
+
options?: VitestOptions,
154
+
)
155
+
/**
156
+
* @deprecated The `mode` argument is no longer used. Use `new Vitest(cliOptions, options)` instead.
157
+
*/
158
+
constructor(
159
+
mode: VitestRunMode,
160
+
cliOptions: UserConfig,
161
+
options?: VitestOptions,
162
+
)
163
+
constructor(
164
+
modeOrCliOptions: VitestRunMode | UserConfig,
165
+
cliOptionsOrOptions?: UserConfig | VitestOptions,
166
+
maybeOptions?: VitestOptions,
150
167
) {
168
+
let cliOptions: UserConfig
169
+
let options: VitestOptions
170
+
if (typeof modeOrCliOptions === 'string') {
171
+
cliOptions = cliOptionsOrOptions as UserConfig
172
+
options = maybeOptions ?? {}
173
+
}
174
+
else {
175
+
cliOptions = modeOrCliOptions
176
+
options = (cliOptionsOrOptions as VitestOptions) ?? {}
177
+
}
151
178
this._cliOptions = cliOptions
152
179
this.logger = new Logger(this, options.stdout, options.stderr)
153
180
this.packageInstaller = options.packageInstaller || new VitestPackageInstaller()
···
306
333
}
307
334
catch { }
308
335
309
-
const projects = await this.resolveProjects(this._cliOptions)
310
-
this.projects = projects
336
+
this.projects = await this.resolveProjects(this._cliOptions)
337
+
if (this._cliOptions.benchmarkOnly) {
338
+
this.projects = this.projects.filter(c => c.config.benchmark.enabled)
339
+
}
311
340
312
-
await Promise.all(projects.flatMap((project) => {
341
+
await Promise.all(this.projects.flatMap((project) => {
313
342
const hooks = project.vite.config.getSortedPluginHooks('configureVitest')
314
343
return hooks.map(hook => hook({
315
344
project,
···
356
385
populateProjectsTags(this.coreWorkspaceProject, this.projects)
357
386
}
358
387
359
-
this.reporters = resolved.mode === 'benchmark'
360
-
? await createBenchmarkReporters(toArray(resolved.benchmark?.reporters), this.runner)
361
-
: await createReporters(resolved.reporters, this)
388
+
this.reporters = await createReporters(resolved.reporters, this)
362
389
363
390
await this._fsCache.ensureCacheIntegrity()
364
391
···
567
594
if (!project) {
568
595
return []
569
596
}
570
-
return resolveBrowserProjects(this, new Set([project.name]), [project])
597
+
return resolveDefaultProjects(this, new Set([project.name]), [project])
571
598
}
572
599
573
600
/**
···
797
824
})
798
825
799
826
if (!this.config.watch || !(this.config.changed || this.config.related?.length)) {
800
-
throw new FilesNotFoundError(this.mode)
827
+
throw new FilesNotFoundError()
801
828
}
802
829
}
803
830
···
1042
1069
/** @default os.availableParallelism() */
1043
1070
concurrency?: number
1044
1071
}): Promise<TestModule[]> {
1045
-
if (this.mode !== 'test') {
1046
-
throw new Error(`The \`experimental_parseSpecifications\` does not support "${this.mode}" mode.`)
1047
-
}
1048
1072
const concurrency = options?.concurrency ?? (typeof os.availableParallelism === 'function'
1049
1073
? os.availableParallelism()
1050
1074
: os.cpus().length)
···
1084
1108
}
1085
1109
1086
1110
public async experimental_parseSpecification(specification: TestSpecification): Promise<TestModule> {
1087
-
if (this.mode !== 'test') {
1088
-
throw new Error(`The \`experimental_parseSpecification\` does not support "${this.mode}" mode.`)
1089
-
}
1090
1111
const file = await astCollectTests(specification.project, specification.moduleId).catch((error) => {
1091
1112
return createFailedFileTask(specification.project, specification.moduleId, error)
1092
1113
})
+31
-5
packages/vitest/src/node/create.ts
+31
-5
packages/vitest/src/node/create.ts
···
16
16
import { createViteServer } from './vite'
17
17
18
18
export async function createVitest(
19
+
options: CliOptions,
20
+
viteOverrides?: ViteUserConfig,
21
+
vitestOptions?: VitestOptions,
22
+
): Promise<Vitest>
23
+
/**
24
+
* @deprecated The `mode` argument is no longer used. Use `createVitest(options, viteOverrides?, vitestOptions?)` instead.
25
+
*/
26
+
export async function createVitest(
19
27
mode: VitestRunMode,
20
28
options: CliOptions,
21
-
viteOverrides: ViteUserConfig = {},
22
-
vitestOptions: VitestOptions = {},
29
+
viteOverrides?: ViteUserConfig,
30
+
vitestOptions?: VitestOptions,
31
+
): Promise<Vitest>
32
+
export async function createVitest(
33
+
modeOrOptions: VitestRunMode | CliOptions,
34
+
optionsOrViteOverrides: CliOptions | ViteUserConfig = {},
35
+
viteOverridesOrVitestOptions: ViteUserConfig | VitestOptions = {},
36
+
maybeVitestOptions: VitestOptions = {},
23
37
): Promise<Vitest> {
24
-
const ctx = new Vitest(mode, deepClone(options), vitestOptions)
38
+
let options: CliOptions
39
+
let viteOverrides: ViteUserConfig
40
+
let vitestOptions: VitestOptions
41
+
if (typeof modeOrOptions === 'string') {
42
+
options = optionsOrViteOverrides as CliOptions
43
+
viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig
44
+
vitestOptions = maybeVitestOptions
45
+
}
46
+
else {
47
+
options = modeOrOptions
48
+
viteOverrides = optionsOrViteOverrides as ViteUserConfig
49
+
vitestOptions = viteOverridesOrVitestOptions as VitestOptions
50
+
}
51
+
const ctx = new Vitest(deepClone(options), vitestOptions)
25
52
const root = slash(resolve(options.root || process.cwd()))
26
53
27
54
const configPath
···
38
65
const config: ViteInlineConfig = {
39
66
configFile: configPath,
40
67
configLoader: options.configLoader,
41
-
// this will make "mode": "test" | "benchmark" inside defineConfig
42
-
mode: options.mode || mode,
68
+
mode: options.mode || 'test',
43
69
plugins: await VitestPlugin(restOptions, ctx),
44
70
}
45
71
+2
-2
packages/vitest/src/node/errors.ts
+2
-2
packages/vitest/src/node/errors.ts
+4
-4
packages/vitest/src/node/logger.ts
+4
-4
packages/vitest/src/node/logger.ts
···
169
169
const config = this.ctx.config
170
170
171
171
if (config.watch && (config.changed || config.related?.length)) {
172
-
this.log(`No affected ${config.mode} files found\n`)
172
+
this.log(`No affected test files found\n`)
173
173
}
174
174
else if (config.watch) {
175
175
this.log(
176
-
c.red(`No ${config.mode} files found. You can change the file name pattern by pressing "p"\n`),
176
+
c.red(`No test files found. You can change the file name pattern by pressing "p"\n`),
177
177
)
178
178
}
179
179
else {
180
180
if (config.passWithNoTests) {
181
-
this.log(`No ${config.mode} files found, exiting with code 0\n`)
181
+
this.log(`No test files found, exiting with code 0\n`)
182
182
}
183
183
else {
184
184
this.error(
185
-
c.red(`No ${config.mode} files found, exiting with code 1\n`),
185
+
c.red(`No test files found, exiting with code 1\n`),
186
186
)
187
187
}
188
188
}
+4
-1
packages/vitest/src/node/project.ts
+4
-1
packages/vitest/src/node/project.ts
···
28
28
import { setup } from '../api/setup'
29
29
import { createDefinesScript } from '../utils/config-helpers'
30
30
import { NativeModuleRunner } from '../utils/nativeModuleRunner'
31
+
import { BenchmarkManager } from './benchmark'
31
32
import { isBrowserEnabled, resolveConfig } from './config/resolveConfig'
32
33
import { serializeConfig } from './config/serializeConfig'
33
34
import { createFetchModuleFunction } from './environments/fetchModule'
···
62
63
* Temporary directory for the project. This is unique for each project. Vitest stores transformed content here.
63
64
*/
64
65
public readonly tmpDir: string
66
+
67
+
public readonly benchmark: BenchmarkManager = new BenchmarkManager(this)
65
68
66
69
/** @internal */ typechecker?: Typechecker
67
70
/** @internal */ _config?: ResolvedConfig
···
731
734
}
732
735
733
736
/** @internal */
734
-
static _cloneBrowserProject(parent: TestProject, config: ResolvedConfig): TestProject {
737
+
static _cloneTestProject(parent: TestProject, config: ResolvedConfig): TestProject {
735
738
const clone = new TestProject(parent.vitest, undefined, parent.tmpDir)
736
739
clone.runner = parent.runner
737
740
clone._vite = parent._vite
+25
-12
packages/vitest/src/node/test-run.ts
+25
-12
packages/vitest/src/node/test-run.ts
···
1
1
import type {
2
2
File as RunnerTestFile,
3
+
TaskEventData,
3
4
TaskEventPack,
4
5
TaskResultPack,
5
6
TaskUpdateEvent,
7
+
TestArtifact,
6
8
TestAttachment,
9
+
TestBenchmark,
7
10
} from '@vitest/runner'
8
-
import type { TaskEventData, TestArtifact } from '@vitest/runner/types/tasks'
9
11
import type { SerializedError } from '@vitest/utils'
10
12
import type { UserConsoleLog } from '../types/general'
11
13
import type { Vitest } from './core'
···
55
57
await this.vitest.report('onUserConsoleLog', log)
56
58
}
57
59
58
-
async recordArtifact<Artifact extends TestArtifact>(testId: string, artifact: Artifact): Promise<Artifact> {
59
-
const task = this.vitest.state.idMap.get(testId)
60
-
const entity = task && this.vitest.state.getReportedEntity(task)
60
+
async recordBenchmark(testId: string, benchmark: TestBenchmark): Promise<void> {
61
+
const testCase = this.getTestCaseById(testId, 'Benchmark')
62
+
testCase.task.benchmarks.push(benchmark)
63
+
await this.vitest.report('onTestCaseBenchmark', testCase, benchmark)
64
+
}
61
65
62
-
assert(task && entity, `Entity must be found for task ${task?.name || testId}`)
63
-
assert(entity.type === 'test', `Artifacts can only be recorded on a test, instead got ${entity.type}`)
66
+
async recordArtifact<Artifact extends TestArtifact>(testId: string, artifact: Artifact): Promise<Artifact> {
67
+
const testCase = this.getTestCaseById(testId, 'Artifact')
64
68
65
69
// annotations won't resolve as artifacts for backwards compatibility until next major
66
70
if (artifact.type === 'internal:annotation') {
67
-
await this.resolveTestAttachment(entity, artifact.annotation.attachment, artifact.annotation.message)
71
+
await this.resolveTestAttachment(testCase, artifact.annotation.attachment, artifact.annotation.message)
68
72
69
-
entity.task.annotations.push(artifact.annotation)
73
+
testCase.task.annotations.push(artifact.annotation)
70
74
71
-
await this.vitest.report('onTestCaseAnnotate', entity, artifact.annotation)
75
+
await this.vitest.report('onTestCaseAnnotate', testCase, artifact.annotation)
72
76
73
77
return artifact
74
78
}
75
79
76
80
if (Array.isArray(artifact.attachments)) {
77
81
await Promise.all(
78
-
artifact.attachments.map(attachment => this.resolveTestAttachment(entity, attachment)),
82
+
artifact.attachments.map(attachment => this.resolveTestAttachment(testCase, attachment)),
79
83
)
80
84
}
81
85
82
-
entity.task.artifacts.push(artifact)
86
+
testCase.task.artifacts.push(artifact)
83
87
84
-
await this.vitest.report('onTestCaseArtifactRecord', entity, artifact)
88
+
await this.vitest.report('onTestCaseArtifactRecord', testCase, artifact)
85
89
86
90
return artifact
87
91
}
···
100
104
// "onTaskUpdate" in parallel with others or before all or after all?
101
105
// TODO: error handling - what happens if custom reporter throws an error?
102
106
await this.vitest.report('onTaskUpdate', update, events)
107
+
}
108
+
109
+
private getTestCaseById(testId: string, recordType: string) {
110
+
const task = this.vitest.state.idMap.get(testId)
111
+
const entity = task && this.vitest.state.getReportedEntity(task)
112
+
113
+
assert(task && entity, `Entity must be found for task ${task?.name || testId}`)
114
+
assert(entity.type === 'test', `${recordType} can only be recorded on a test, instead got ${entity.type}`)
115
+
return entity
103
116
}
104
117
105
118
async end(specifications: TestSpecification[], errors: unknown[], coverage?: unknown): Promise<void> {
+12
-12
packages/vitest/src/public/index.ts
+12
-12
packages/vitest/src/public/index.ts
···
34
34
35
35
export { vi, vitest } from '../integrations/vi'
36
36
export type { VitestUtils } from '../integrations/vi'
37
-
export { bench } from '../runtime/benchmark'
37
+
export type {
38
+
Bench,
39
+
BenchCompareOptions,
40
+
BenchFnOptions,
41
+
BenchFromSource,
42
+
BenchRegistration,
43
+
BenchResult,
44
+
BenchStorage,
45
+
} from '../runtime/benchmark'
38
46
39
47
export type {
40
48
RuntimeConfig,
···
45
53
46
54
export { VitestEvaluatedModules as EvaluatedModules } from '../runtime/moduleRunner/evaluatedModules'
47
55
48
-
export { NodeBenchmarkRunner as BenchmarkRunner } from '../runtime/runners/benchmark'
49
56
export { TestRunner } from '../runtime/runners/test'
50
-
export type {
51
-
BenchFactory,
52
-
BenchFunction,
53
-
Benchmark,
54
-
BenchmarkAPI,
55
-
BenchmarkResult,
56
-
BenchOptions,
57
-
BenchTask,
58
-
BenchTaskResult,
59
-
} from '../runtime/types/benchmark'
60
57
export { assertType } from '../typecheck/assertType'
61
58
62
59
export type { AssertType } from '../typecheck/assertType'
···
117
114
test,
118
115
} from '@vitest/runner'
119
116
export type {
117
+
BaselineData,
120
118
ImportDuration,
121
119
OnTestFailedHandler,
122
120
OnTestFinishedHandler,
···
144
142
TestArtifactLocation,
145
143
TestArtifactRegistry,
146
144
TestAttachment,
145
+
TestBenchmark,
146
+
TestBenchmarkTask,
147
147
TestContext,
148
148
TestFunction,
149
149
TestOptions,
-4
packages/vitest/src/public/node.ts
-4
packages/vitest/src/public/node.ts
···
44
44
45
45
export {
46
46
AgentReporter,
47
-
BenchmarkReporter,
48
-
BenchmarkReportsMap,
49
47
DefaultReporter,
50
48
DotReporter,
51
49
GithubActionsReporter,
···
56
54
ReportersMap,
57
55
TapFlatReporter,
58
56
TapReporter,
59
-
VerboseBenchmarkReporter,
60
57
VerboseReporter,
61
58
} from '../node/reporters'
62
59
export type {
63
60
BaseReporter,
64
-
BenchmarkBuiltinReporters,
65
61
BuiltinReporterOptions,
66
62
BuiltinReporters,
67
63
JsonAssertionResult,
+500
-53
packages/vitest/src/runtime/benchmark.ts
+500
-53
packages/vitest/src/runtime/benchmark.ts
···
1
-
import type { Test } from '@vitest/runner'
2
-
import type { BenchFunction, BenchmarkAPI, BenchOptions } from './types/benchmark'
3
-
import { getCurrentSuite } from '@vitest/runner'
4
-
import { createChainable } from '@vitest/runner/utils'
5
-
import { noop } from '@vitest/utils/helpers'
1
+
import type { BaselineData, Test, TestBenchmark, TestBenchmarkTask } from '@vitest/runner'
2
+
import type {
3
+
BenchOptions as BenchCompareOptions,
4
+
Fn,
5
+
FnOptions,
6
+
TaskResultCompleted,
7
+
TaskResultRuntimeInfo,
8
+
TaskResultTimestampProviderInfo,
9
+
Task as TinybenchTask,
10
+
} from 'tinybench'
11
+
import type { SerializedConfig } from './config'
12
+
import { isAbsolute, relative } from 'pathe'
13
+
import { Bench as Tinybench } from 'tinybench'
14
+
import c from 'tinyrainbow'
15
+
import { rpc } from './rpc'
16
+
import { TestRunner } from './runners/test'
6
17
import { getWorkerState } from './utils'
7
18
8
-
const benchFns = new WeakMap<Test, BenchFunction>()
9
-
const benchOptsMap = new WeakMap()
19
+
const now = globalThis.performance
20
+
? globalThis.performance.now.bind(globalThis.performance)
21
+
: Date.now
10
22
11
-
export function getBenchOptions(key: Test): BenchOptions {
12
-
return benchOptsMap.get(key)
23
+
const kRegistration: unique symbol = Symbol('registration')
24
+
const kFromSource: unique symbol = Symbol('fromSource')
25
+
const kPerProject: unique symbol = Symbol('perProject')
26
+
const kWriteResult: unique symbol = Symbol('writeResult')
27
+
export const kFinalize: unique symbol = Symbol('finalize')
28
+
29
+
type ExtractBenchNames<T extends BenchRegistration<any>[]> = Exclude<{
30
+
[K in keyof T]: T[K] extends BenchRegistration<infer N> ? N : never
31
+
}[number], never>
32
+
33
+
// We throw an error if benchmark did not complete, so it will always be TaskResultCompleted
34
+
export type BenchResult = TaskResultCompleted & TaskResultRuntimeInfo & TaskResultTimestampProviderInfo
35
+
36
+
export interface BenchStorage<T extends string> {
37
+
get: (name: T) => BenchResult
13
38
}
14
39
15
-
export function getBenchFn(key: Test): BenchFunction {
16
-
return benchFns.get(key)!
40
+
export type { BenchOptions as BenchCompareOptions } from 'tinybench'
41
+
42
+
/**
43
+
* Options accepted by `bench(name, options, fn)`. Extends tinybench's
44
+
* `FnOptions` with Vitest-specific fields.
45
+
*/
46
+
export interface BenchFnOptions extends FnOptions {
47
+
/**
48
+
* Path (relative to the project root) where the benchmark result is written
49
+
* after a successful run. The string `${projectName}` is substituted with
50
+
* the current project name. Absolute paths are accepted as long as they
51
+
* resolve inside the project root.
52
+
*/
53
+
writeResult?: string
54
+
/**
55
+
* Mark this benchmark as a per-project entry. Per-project tasks still appear
56
+
* in the inline comparison table for the current run, and Vitest additionally
57
+
* collects them across projects and prints a single cross-project table at
58
+
* the end of the run.
59
+
*/
60
+
perProject?: boolean
17
61
}
18
62
19
-
export const bench: BenchmarkAPI = createBenchmark(function (
20
-
name,
21
-
fn: BenchFunction = noop,
22
-
options: BenchOptions = {},
23
-
) {
24
-
if (getWorkerState().config.mode !== 'benchmark') {
25
-
throw new Error('`bench()` is only available in benchmark mode.')
63
+
export interface BenchRegistration<Name extends string> {
64
+
name: Name
65
+
/**
66
+
* The benchmark function. Absent for registrations created via `bench.from()`.
67
+
*/
68
+
fn?: Fn
69
+
/**
70
+
* Per-benchmark options (`beforeEach`, `beforeAll`, etc.). Absent for
71
+
* registrations created via `bench.from()`.
72
+
*/
73
+
fnOpts?: FnOptions
74
+
/**
75
+
* @internal
76
+
*/
77
+
[kRegistration]: true
78
+
run: (options?: BenchCompareOptions) => Promise<BenchResult>
79
+
}
80
+
81
+
interface BenchCompare {
82
+
<Args extends BenchRegistration<any>[]>(...args: Args): Promise<BenchStorage<ExtractBenchNames<Args>>>
83
+
<Args extends BenchRegistration<any>[]>(...args: [...Args, BenchCompareOptions]): Promise<BenchStorage<ExtractBenchNames<Args>>>
84
+
}
85
+
86
+
interface BenchFactory {
87
+
<Name extends string>(name: Name | Function, fn: Fn): BenchRegistration<Name>
88
+
<Name extends string>(name: Name | Function, options: BenchFnOptions, fn: Fn): BenchRegistration<Name>
89
+
}
90
+
91
+
export interface BenchFromSource {
92
+
(): BaselineData | Promise<BaselineData>
93
+
}
94
+
95
+
interface BenchFrom {
96
+
<Name extends string>(name: Name | Function, source: string | BenchFromSource): BenchRegistration<Name>
97
+
}
98
+
99
+
export interface Bench extends BenchFactory {
100
+
compare: BenchCompare
101
+
from: BenchFrom
102
+
/** @internal */
103
+
[kFinalize]: () => void
104
+
}
105
+
106
+
interface RunnableRegistration<Name extends string> extends BenchRegistration<Name> {
107
+
fn: Fn
108
+
fnOpts?: FnOptions
109
+
[kWriteResult]?: string
110
+
[kPerProject]?: true
111
+
}
112
+
113
+
interface FromRegistration<Name extends string> extends BenchRegistration<Name> {
114
+
[kFromSource]: string | BenchFromSource
115
+
}
116
+
117
+
function isFromRegistration(reg: BenchRegistration<any>): reg is FromRegistration<any> {
118
+
return kFromSource in reg
119
+
}
120
+
121
+
function substitutePath(template: string, projectName: string | undefined): string {
122
+
return template.replace(/\$\{projectName\}/g, projectName ?? '')
123
+
}
124
+
125
+
export function createBench(test: Test, config: SerializedConfig): Bench {
126
+
let benchIdx = 0
127
+
const pending = new Set<BenchRegistration<any>>()
128
+
const createTinybench = (options?: BenchCompareOptions) => {
129
+
const currentIndex = ++benchIdx
130
+
return new Tinybench({
131
+
signal: test.context.signal,
132
+
name: `${test.fullTestName} ${currentIndex}`,
133
+
retainSamples: config.benchmark.retainSamples,
134
+
...options,
135
+
now,
136
+
})
26
137
}
27
138
28
-
const task = getCurrentSuite().task(formatName(name), {
29
-
...this,
30
-
meta: {
31
-
benchmark: true,
32
-
},
139
+
const resolveTemplate = (template: string) => substitutePath(template, config.benchmark.projectName)
140
+
141
+
const resolveFromSource = async (source: string | BenchFromSource): Promise<BaselineData> => {
142
+
if (typeof source === 'function') {
143
+
return source()
144
+
}
145
+
const resolved = resolveTemplate(source)
146
+
const data = await rpc().readBenchmarkResult(resolved)
147
+
if (data == null) {
148
+
throw new Error(`\`bench.from()\` could not find a result file at "${resolved}". Run the source benchmark first to create it.`)
149
+
}
150
+
return data
151
+
}
152
+
153
+
const taskFromBaseline = (
154
+
name: string,
155
+
data: BaselineData,
156
+
): TestBenchmarkTask => ({
157
+
name,
158
+
latency: data.latency,
159
+
throughput: data.throughput,
160
+
period: data.period,
161
+
totalTime: data.totalTime,
162
+
rank: 0,
163
+
fromStore: true,
33
164
})
34
-
benchFns.set(task, fn)
35
-
benchOptsMap.set(task, options)
36
-
// vitest runner sets mode to `todo` if handler is not passed down
37
-
// but we store handler separately
38
-
if (!this.todo && task.mode === 'todo') {
39
-
task.mode = 'run'
165
+
166
+
const createCompareStorage = <T extends string>(
167
+
bench: Tinybench,
168
+
fromResults?: Map<string, BaselineData>,
169
+
): BenchStorage<T> => {
170
+
return {
171
+
get(name: T) {
172
+
const stored = fromResults?.get(name)
173
+
if (stored) {
174
+
return stored as BenchResult
175
+
}
176
+
const task = bench.getTask(name)
177
+
if (!task) {
178
+
throw new Error(`task "${name}" was not defined`)
179
+
}
180
+
return task.result as BenchResult
181
+
},
182
+
}
40
183
}
41
-
})
42
184
43
-
function createBenchmark(
44
-
fn: (
45
-
this: Record<'skip' | 'only' | 'todo', boolean | undefined>,
46
-
name: string | Function,
47
-
fn?: BenchFunction,
48
-
options?: BenchOptions,
49
-
) => void,
50
-
) {
51
-
const benchmark = createChainable(
52
-
['skip', 'only', 'todo'],
53
-
fn,
54
-
) as BenchmarkAPI
185
+
interface TaskMeta { perProject?: true }
55
186
56
-
benchmark.skipIf = (condition: any) =>
57
-
(condition ? benchmark.skip : benchmark) as BenchmarkAPI
58
-
benchmark.runIf = (condition: any) =>
59
-
(condition ? benchmark : benchmark.skip) as BenchmarkAPI
187
+
const serializeBenchmark = (
188
+
tinybenchTasks: TinybenchTask[],
189
+
name: string | undefined,
190
+
taskMeta?: Map<string, TaskMeta>,
191
+
fromTasks?: TestBenchmarkTask[],
192
+
): TestBenchmark => {
193
+
const tasks: TestBenchmarkTask[] = tinybenchTasks.map((t) => {
194
+
const result = t.result
195
+
if (result.state === 'errored') {
196
+
throw result.error
197
+
}
198
+
if (result.state !== 'completed') {
199
+
throw new Error(`task "${t.name}" did not complete: received "${result.state}"`)
200
+
}
201
+
return {
202
+
name: t.name,
203
+
latency: result.latency,
204
+
throughput: result.throughput,
205
+
period: result.period,
206
+
totalTime: result.totalTime,
207
+
rank: 0,
208
+
...taskMeta?.get(t.name),
209
+
}
210
+
})
211
+
if (fromTasks) {
212
+
tasks.push(...fromTasks)
213
+
}
214
+
tasks.sort((a, b) => a.latency.mean - b.latency.mean)
215
+
tasks.forEach((task, idx) => {
216
+
task.rank = idx + 1
217
+
})
218
+
return {
219
+
name: name || test.fullTestName,
220
+
tasks,
221
+
}
222
+
}
60
223
61
-
return benchmark as BenchmarkAPI
224
+
const recordBenchmark = async (
225
+
tinybenchTasks: TinybenchTask[],
226
+
name: string | undefined,
227
+
taskMeta?: Map<string, TaskMeta>,
228
+
fromTasks?: TestBenchmarkTask[],
229
+
) => {
230
+
const serializedBenchmark = serializeBenchmark(tinybenchTasks, name, taskMeta, fromTasks)
231
+
test.benchmarks.push(serializedBenchmark)
232
+
await rpc().onTestBenchmark(test.id, serializedBenchmark)
233
+
}
234
+
235
+
const writeResultArtifact = async (template: string, result: BenchResult) => {
236
+
const resolved = resolveTemplate(template)
237
+
const data: BaselineData = {
238
+
latency: result.latency,
239
+
throughput: result.throughput,
240
+
period: result.period,
241
+
totalTime: result.totalTime,
242
+
}
243
+
await rpc().writeBenchmarkResult(resolved, data)
244
+
}
245
+
246
+
const runBenchmarks = async (tinybench: Tinybench) => {
247
+
const workerState = getWorkerState()
248
+
const getterTracker = workerState.getterTracker
249
+
getterTracker?.resetInvocations()
250
+
try {
251
+
return await TestRunner.runBenchmarks(tinybench)
252
+
}
253
+
finally {
254
+
const excessiveInvocations = config.benchmark.suppressExportGetterWarnings
255
+
? undefined
256
+
: getterTracker?.getExcessiveInvocations()
257
+
if (excessiveInvocations?.length) {
258
+
const entries = excessiveInvocations
259
+
.map(({ moduleId, exportName }) => ` - ${formatModuleId(moduleId, workerState.config.root)} > ${exportName}`)
260
+
.join('\n')
261
+
console.warn(
262
+
[
263
+
c.yellow(c.bold('Benchmark Warning')),
264
+
`Benchmark ${c.bold(`"${tinybench.name}"`)} accessed module export getters too many times.`,
265
+
'',
266
+
'This can make results unreliable because export getters add overhead.',
267
+
'See https://vitest.dev/guide/benchmarking#module-runner-overhead',
268
+
'',
269
+
'Tracked exports:',
270
+
entries,
271
+
].join('\n'),
272
+
)
273
+
}
274
+
}
275
+
}
276
+
277
+
const runSingle = async (
278
+
name: string,
279
+
fn: Fn,
280
+
fnOpts: FnOptions | undefined,
281
+
options: BenchCompareOptions | undefined,
282
+
meta: TaskMeta | undefined,
283
+
writeResult: string | undefined,
284
+
): Promise<BenchResult> => {
285
+
const tinybench = createTinybench(options).add(name, fn, fnOpts)
286
+
const tasks = await runBenchmarks(tinybench)
287
+
const task = tinybench.getTask(name)!
288
+
if (task.result.state === 'errored') {
289
+
throw task.result.error
290
+
}
291
+
await recordBenchmark(tasks, tinybench.name, meta ? new Map([[name, meta]]) : undefined)
292
+
if (writeResult) {
293
+
await writeResultArtifact(writeResult, task.result as BenchResult)
294
+
}
295
+
return task.result as BenchResult
296
+
}
297
+
298
+
const runFrom = async (
299
+
name: string,
300
+
source: string | BenchFromSource,
301
+
): Promise<BenchResult> => {
302
+
const data = await resolveFromSource(source)
303
+
const benchmark: TestBenchmark = {
304
+
name: test.fullTestName,
305
+
tasks: [{ ...taskFromBaseline(name, data), rank: 1 }],
306
+
}
307
+
test.benchmarks.push(benchmark)
308
+
await rpc().onTestBenchmark(test.id, benchmark)
309
+
return data as BenchResult
310
+
}
311
+
312
+
const bench: Bench = (nameOrFunction: string | Function, a: Fn | BenchFnOptions, b?: Fn | BenchFnOptions) => {
313
+
validateBenchmarkProject(config)
314
+
const { fn, fnOpts, writeResult, perProject } = normalizeBenchArgs(a, b)
315
+
const name = typeof nameOrFunction === 'function' ? nameOrFunction.name || '<anonymous>' : nameOrFunction
316
+
const meta: TaskMeta | undefined = perProject ? { perProject: true } : undefined
317
+
const registration: RunnableRegistration<string> = {
318
+
[kRegistration]: true,
319
+
name,
320
+
fn,
321
+
fnOpts,
322
+
run: (options?: BenchCompareOptions) => {
323
+
pending.delete(registration)
324
+
return runSingle(name, fn, fnOpts, options, meta, writeResult)
325
+
},
326
+
}
327
+
if (perProject) {
328
+
registration[kPerProject] = true
329
+
}
330
+
if (writeResult) {
331
+
registration[kWriteResult] = writeResult
332
+
}
333
+
pending.add(registration)
334
+
return registration
335
+
}
336
+
337
+
bench.from = <Name extends string>(nameOrFunction: Name | Function, source: string | BenchFromSource): BenchRegistration<Name> => {
338
+
validateBenchmarkProject(config)
339
+
if (typeof nameOrFunction !== 'string' && typeof nameOrFunction !== 'function') {
340
+
throw new TypeError('`bench.from()` requires a name (string or named function) as its first argument.')
341
+
}
342
+
if (typeof source !== 'string' && typeof source !== 'function') {
343
+
throw new TypeError('`bench.from()` expects a string path or a function returning the result data as its second argument.')
344
+
}
345
+
const name = (typeof nameOrFunction === 'function' ? nameOrFunction.name || '<anonymous>' : nameOrFunction) as Name
346
+
const registration: FromRegistration<Name> = {
347
+
[kRegistration]: true,
348
+
[kFromSource]: source,
349
+
name,
350
+
run: () => {
351
+
pending.delete(registration)
352
+
return runFrom(name, source)
353
+
},
354
+
}
355
+
pending.add(registration)
356
+
return registration
357
+
}
358
+
359
+
bench.compare = async (...args) => {
360
+
validateBenchmarkProject(config)
361
+
362
+
// extract optional trailing BenchCompareOptions argument
363
+
const lastArg = args[args.length - 1]
364
+
const isOptions = lastArg != null && typeof lastArg === 'object' && !(kRegistration in lastArg)
365
+
const benchOptions = isOptions ? args.pop() as BenchCompareOptions : undefined
366
+
const registrations = args as BenchRegistration<any>[]
367
+
368
+
// Mark every passed-in registration as consumed before validation so a
369
+
// throwing `bench.compare()` (wrong arity, wrong shape) doesn't also
370
+
// trigger the unrun-bench warning — the user's intent was to consume them.
371
+
for (const reg of registrations) {
372
+
if (reg != null && typeof reg === 'object' && kRegistration in reg) {
373
+
pending.delete(reg)
374
+
}
375
+
}
376
+
377
+
if (registrations.length < 2) {
378
+
throw new SyntaxError(`\`bench.compare()\` requires at least 2 benchmarks, received ${registrations.length} instead. ${registrations.length === 1 ? 'Consider calling `bench().run()`. ' : 'Define benchmarks by calling `bench()`. '}See https://vitest.dev/guide/benchmarking#comparing-benchmarks`)
379
+
}
380
+
for (const reg of registrations) {
381
+
if (reg == null || typeof reg !== 'object' || !(kRegistration in reg)) {
382
+
throw new SyntaxError('`bench.compare()` expects every argument to be the return value of `bench` or `bench.from`.')
383
+
}
384
+
}
385
+
386
+
const runnable: RunnableRegistration<any>[] = []
387
+
const fromEntries: FromRegistration<any>[] = []
388
+
for (const reg of registrations) {
389
+
if (isFromRegistration(reg)) {
390
+
fromEntries.push(reg)
391
+
}
392
+
else {
393
+
runnable.push(reg as RunnableRegistration<any>)
394
+
}
395
+
}
396
+
397
+
const taskMeta = new Map<string, TaskMeta>()
398
+
for (const reg of runnable) {
399
+
if (reg[kPerProject]) {
400
+
taskMeta.set(reg.name, { perProject: true })
401
+
}
402
+
}
403
+
404
+
const fromResults = new Map<string, BaselineData>()
405
+
const fromTasks: TestBenchmarkTask[] = []
406
+
if (fromEntries.length > 0) {
407
+
const resolved = await Promise.all(
408
+
fromEntries.map(async (reg) => {
409
+
const data = await resolveFromSource(reg[kFromSource])
410
+
return { reg, data }
411
+
}),
412
+
)
413
+
for (const { reg, data } of resolved) {
414
+
fromResults.set(reg.name, data)
415
+
fromTasks.push(taskFromBaseline(reg.name, data))
416
+
}
417
+
}
418
+
419
+
const tinybench = createTinybench(benchOptions)
420
+
runnable.forEach((reg) => {
421
+
tinybench.add(reg.name, reg.fn, reg.fnOpts)
422
+
})
423
+
424
+
let tasks: TinybenchTask[] = []
425
+
if (runnable.length > 0) {
426
+
tasks = await runBenchmarks(tinybench)
427
+
const errors = tinybench.tasks
428
+
.filter(task => task.result.state === 'errored')
429
+
.map(task => (task.result as any).error)
430
+
if (errors.length > 0) {
431
+
throw new AggregateError(errors, 'Some benchmarks failed')
432
+
}
433
+
}
434
+
435
+
await recordBenchmark(tasks, tinybench.name, taskMeta, fromTasks)
436
+
437
+
// write artifacts for every runnable registration that requested it. We
438
+
// do this after recording so a write failure can't be confused with a
439
+
// benchmark failure in the reporter output.
440
+
await Promise.all(
441
+
runnable
442
+
.filter(reg => reg[kWriteResult] != null)
443
+
.map((reg) => {
444
+
const task = tinybench.getTask(reg.name)!
445
+
return writeResultArtifact(reg[kWriteResult]!, task.result as BenchResult)
446
+
}),
447
+
)
448
+
449
+
return createCompareStorage(tinybench, fromResults)
450
+
}
451
+
452
+
bench[kFinalize] = () => {
453
+
if (pending.size === 0) {
454
+
return
455
+
}
456
+
const names = [...pending].map(reg => `"${reg.name}"`).join(', ')
457
+
pending.clear()
458
+
console.warn(
459
+
[
460
+
c.yellow(c.bold('Benchmark Warning')),
461
+
`Test ${c.bold(`"${test.fullTestName}"`)} registered benchmarks that never ran: ${names}.`,
462
+
'',
463
+
'Call `.run()` on the registration, or pass it to `bench.compare()`.',
464
+
'See https://vitest.dev/guide/benchmarking#defining-a-benchmark',
465
+
].join('\n'),
466
+
)
467
+
}
468
+
469
+
return bench
62
470
}
63
471
64
-
function formatName(name: string | Function) {
65
-
return typeof name === 'string'
66
-
? name
67
-
: typeof name === 'function'
68
-
? name.name || '<anonymous>'
69
-
: String(name)
472
+
function formatModuleId(moduleId: string, root: string): string {
473
+
if (!root || !isAbsolute(moduleId)) {
474
+
return moduleId
475
+
}
476
+
return relative(root, moduleId)
477
+
}
478
+
479
+
function normalizeBenchArgs(
480
+
a: Fn | BenchFnOptions,
481
+
b: Fn | BenchFnOptions | undefined,
482
+
): { fn: Fn; fnOpts: FnOptions | undefined; writeResult: string | undefined; perProject: boolean } {
483
+
if (typeof a === 'function') {
484
+
if (b !== undefined) {
485
+
throw new TypeError('`bench()` does not accept options as the third argument. Pass options as the second argument instead: `bench(name, options, fn)`.')
486
+
}
487
+
return { fn: a, fnOpts: undefined, writeResult: undefined, perProject: false }
488
+
}
489
+
if (typeof b !== 'function') {
490
+
throw new TypeError('`bench()` expects a benchmark function. Call `bench(name, fn)` or `bench(name, options, fn)`.')
491
+
}
492
+
// Strip vitest-specific fields only when present so we don't allocate a new
493
+
// object — preserving referential identity matters: users inspect
494
+
// `registration.fnOpts` and tinybench's `add` sees the same object the
495
+
// caller passed in.
496
+
if (a.writeResult === undefined && a.perProject === undefined) {
497
+
return { fn: b, fnOpts: a as FnOptions, writeResult: undefined, perProject: false }
498
+
}
499
+
const { writeResult, perProject, ...fnOpts } = a
500
+
return {
501
+
fn: b,
502
+
fnOpts: Object.keys(fnOpts).length > 0 ? fnOpts as FnOptions : undefined,
503
+
writeResult,
504
+
perProject: perProject ?? false,
505
+
}
506
+
}
507
+
508
+
function validateBenchmarkProject(config: SerializedConfig) {
509
+
if (!config.benchmark.enabled) {
510
+
throw new Error(
511
+
`Cannot use the \`bench\` test-context fixture within a regular test run. `
512
+
+ `Benchmarks are inherently flaky, so Vitest runs them in a dedicated project based on the \`benchmark.include\` pattern (default \`**/*.{bench,benchmark}.?(c|m)[jt]s?(x)\`). `
513
+
+ `Move this code to a file matched by \`benchmark.include\`, and make sure \`bench\` is destructured from the test context (\`test('...', async ({ bench }) => { ... })\`) — it is not a top-level export of \`vitest\`. `
514
+
+ `See https://vitest.dev/guide/benchmarking#stability`,
515
+
)
516
+
}
70
517
}
+7
-26
packages/vitest/src/runtime/config.ts
+7
-26
packages/vitest/src/runtime/config.ts
···
1
1
import type { Config as FakeTimersConfig } from '@sinonjs/fake-timers'
2
2
import type { PrettyFormatOptions } from '@vitest/pretty-format'
3
-
import type { SequenceHooks, SequenceSetupFiles, SerializableRetry, TestTagDefinition } from '@vitest/runner'
3
+
import type { SequenceHooks, VitestRunnerConfig } from '@vitest/runner'
4
4
import type { SnapshotEnvironment, SnapshotUpdateState } from '@vitest/snapshot'
5
5
import type { SerializedDiffOptions } from '@vitest/utils/diff'
6
6
import type { LabelColor } from '../types/general'
···
8
8
/**
9
9
* Config that tests have access to.
10
10
*/
11
-
export interface SerializedConfig {
12
-
name: string | undefined
11
+
export interface SerializedConfig extends VitestRunnerConfig {
13
12
color?: LabelColor
14
13
globals: boolean
15
14
base: string | undefined
···
18
17
runner: string | undefined
19
18
isolate: boolean
20
19
maxWorkers: number
21
-
mode: 'test' | 'benchmark'
22
20
bail: number | undefined
23
21
environmentOptions?: Record<string, any>
24
-
root: string
25
-
setupFiles: string[]
26
-
passWithNoTests: boolean
27
-
testNamePattern: RegExp | undefined
28
-
allowOnly: boolean
29
-
testTimeout: number
30
-
hookTimeout: number
31
22
clearMocks: boolean
32
23
mockReset: boolean
33
24
restoreMocks: boolean
···
35
26
unstubEnvs: boolean
36
27
// TODO: make optional
37
28
fakeTimers: FakeTimersConfig
38
-
maxConcurrency: number
39
29
defines: Record<string, any>
40
30
expect: {
41
31
requireAssertions?: boolean
···
45
35
}
46
36
}
47
37
printConsoleTrace: boolean | undefined
48
-
sequence: {
49
-
shuffle?: boolean
50
-
concurrent?: boolean
51
-
seed: number
52
-
hooks: SequenceHooks
53
-
setupFiles: SequenceSetupFiles
54
-
}
55
38
deps: {
56
39
web: {
57
40
transformAssets?: boolean
···
84
67
allowWrite: boolean | undefined
85
68
}
86
69
diff: string | SerializedDiffOptions | undefined
87
-
retry: SerializableRetry
88
-
includeTaskLocation: boolean | undefined
89
70
inspect: boolean | string | undefined
90
71
inspectBrk: boolean | string | undefined
91
72
inspector: {
···
130
111
detectAsyncLeaks: boolean
131
112
coverage: SerializedCoverageConfig
132
113
benchmark: {
133
-
includeSamples: boolean
134
-
} | undefined
114
+
enabled: boolean
115
+
retainSamples: boolean
116
+
suppressExportGetterWarnings: boolean
117
+
projectName: string
118
+
}
135
119
serializedDefines: string
136
120
experimental: {
137
121
fsModuleCache: boolean
···
152
136
browserSdkPath?: string
153
137
} | undefined
154
138
}
155
-
tags: TestTagDefinition[]
156
-
tagsFilter: string[] | undefined
157
-
strictTags: boolean
158
139
mergeReportsLabel: string | undefined
159
140
slowTestThreshold: number | undefined
160
141
disableColors: boolean
+37
packages/vitest/src/runtime/getter-tracker.ts
+37
packages/vitest/src/runtime/getter-tracker.ts
···
1
+
export class GetterTracker {
2
+
static EXPORTS_MAX_INVOCATIONS = 1_000_000
3
+
4
+
private invocations = new Map<string, number>()
5
+
private excessiveInvocations = new Map<string, GetterTrackerExport>()
6
+
7
+
public createTracker(
8
+
moduleId: string,
9
+
defineExport: (name: string, getter: () => unknown) => void,
10
+
): (name: string, getter: () => unknown) => void {
11
+
return (name, getter) => {
12
+
const key = `${moduleId}:${name}`
13
+
defineExport(name, () => {
14
+
const count = (this.invocations.get(key) || 0) + 1
15
+
this.invocations.set(key, count)
16
+
if (count > GetterTracker.EXPORTS_MAX_INVOCATIONS && !this.excessiveInvocations.has(key)) {
17
+
this.excessiveInvocations.set(key, { moduleId, exportName: name })
18
+
}
19
+
return getter()
20
+
})
21
+
}
22
+
}
23
+
24
+
public resetInvocations(): void {
25
+
this.invocations.clear()
26
+
this.excessiveInvocations.clear()
27
+
}
28
+
29
+
public getExcessiveInvocations(): GetterTrackerExport[] {
30
+
return [...this.excessiveInvocations.values()]
31
+
}
32
+
}
33
+
34
+
export interface GetterTrackerExport {
35
+
moduleId: string
36
+
exportName: string
37
+
}
+4
packages/vitest/src/runtime/worker.ts
+4
packages/vitest/src/runtime/worker.ts
···
2
2
import type { Traces } from '../utils/traces'
3
3
import type { VitestWorker } from './workers/types'
4
4
import { createStackString, parseStacktrace } from '@vitest/utils/source-map'
5
+
import { GetterTracker } from './getter-tracker'
5
6
import { setupInspect } from './inspector'
6
7
import * as listeners from './listeners'
7
8
import { VitestEvaluatedModules } from './moduleRunner/evaluatedModules'
···
47
48
return createStackString(parseStacktrace(stack))
48
49
},
49
50
metaEnv: createImportMetaEnvProxy(),
51
+
getterTracker: ctx.config.benchmark.enabled && !ctx.config.benchmark.suppressExportGetterWarnings
52
+
? new GetterTracker()
53
+
: undefined,
50
54
} satisfies WorkerGlobalState
51
55
52
56
const methodName = method === 'collect' ? 'collectTests' : 'runTests'
+40
-5
packages/vitest/src/types/global.ts
+40
-5
packages/vitest/src/types/global.ts
···
2
2
import type { Plugin as PrettyFormatPlugin } from '@vitest/pretty-format'
3
3
import type { Test } from '@vitest/runner'
4
4
import type { SnapshotState } from '@vitest/snapshot'
5
-
import type { BenchmarkResult } from '../runtime/types/benchmark'
5
+
import type { Bench, BenchResult } from '../runtime/benchmark'
6
6
import type { UserConsoleLog } from './general'
7
7
8
8
interface SnapshotMatcher<T> {
···
92
92
* await expect(largeData).toMatchFileSnapshot('path/to/snapshot.json');
93
93
*/
94
94
toMatchFileSnapshot: (filepath: string, hint?: string) => Promise<void>
95
+
96
+
/**
97
+
* Asserts that a benchmark result is faster than another benchmark result.
98
+
* Compares mean latency — lower is faster.
99
+
*
100
+
* @example
101
+
* const result = await bench.compare(
102
+
* bench('lib1', () => { lib1() }),
103
+
* bench('lib2', () => { lib2() }),
104
+
* )
105
+
* expect(result.get('lib1')).toBeFasterThan(result.get('lib2'))
106
+
* expect(result.get('lib1')).toBeFasterThan(result.get('lib2'), { delta: 0.1 })
107
+
*/
108
+
toBeFasterThan: (
109
+
expected: BenchResult,
110
+
options?: { delta?: number },
111
+
) => void
112
+
113
+
/**
114
+
* Asserts that a benchmark result is slower than another benchmark result.
115
+
* Compares mean latency — higher is slower.
116
+
*
117
+
* @example
118
+
* const result = await bench.compare(
119
+
* bench('lib1', () => { lib1() }),
120
+
* bench('lib2', () => { lib2() }),
121
+
* )
122
+
* expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'))
123
+
* expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'), { delta: 0.2 })
124
+
*/
125
+
toBeSlowerThan: (
126
+
expected: BenchResult,
127
+
options?: { delta?: number },
128
+
) => void
95
129
}
96
130
}
97
131
···
103
137
* This API is useful for running snapshot tests concurrently because global expect cannot track them.
104
138
*/
105
139
readonly expect: ExpectStatic
140
+
/**
141
+
* Create a benchmark to run. It will be reported after the test is finished.
142
+
* @see {@link https://vitest.dev/guide/benchmarking}
143
+
*/
144
+
readonly bench: Bench
106
145
/** @internal */
107
146
_local: boolean
108
147
}
···
120
159
121
160
interface TaskBase {
122
161
logs?: UserConsoleLog[]
123
-
}
124
-
125
-
interface TaskResult {
126
-
benchmark?: BenchmarkResult
127
162
}
128
163
}
+5
-1
packages/vitest/src/types/rpc.ts
+5
-1
packages/vitest/src/types/rpc.ts
···
1
-
import type { CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner'
1
+
import type { BaselineData, CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact, TestBenchmark } from '@vitest/runner'
2
2
import type { SnapshotResult } from '@vitest/snapshot'
3
3
import type { FetchFunctionOptions, FetchResult } from 'vite/module-runner'
4
4
import type { OTELCarrier } from '../utils/traces'
···
21
21
onQueued: (file: File) => void
22
22
onCollected: (files: File[]) => Promise<void>
23
23
onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
24
+
onTestBenchmark: (testId: string, bench: TestBenchmark) => void
24
25
onTaskArtifactRecord: <Artifact extends TestArtifact>(testId: string, artifact: Artifact) => Promise<Artifact>
25
26
onTaskUpdate: (pack: TaskResultPack[], events: TaskEventPack[]) => Promise<void>
26
27
onCancel: (reason: CancelReason) => void
···
28
29
29
30
snapshotSaved: (snapshot: SnapshotResult) => void
30
31
resolveSnapshotPath: (testPath: string) => string
32
+
33
+
readBenchmarkResult: (relativePath: string) => Promise<BaselineData | null>
34
+
writeBenchmarkResult: (relativePath: string, data: BaselineData) => Promise<void>
31
35
32
36
ensureModuleGraphEntry: (id: string, importer: string) => void
33
37
}
+2
packages/vitest/src/types/worker.ts
+2
packages/vitest/src/types/worker.ts
···
2
2
import type { BirpcReturn } from 'birpc'
3
3
import type { EvaluatedModules } from 'vite/module-runner'
4
4
import type { SerializedConfig } from '../runtime/config'
5
+
import type { GetterTracker } from '../runtime/getter-tracker'
5
6
import type { Traces } from '../utils/traces'
6
7
import type { Environment } from './environment'
7
8
import type { RunnerRPC, RuntimeRPC } from './rpc'
···
74
75
evaluatedModules: EvaluatedModules
75
76
resolvingModules: Set<string>
76
77
moduleExecutionInfo: Map<string, any>
78
+
getterTracker?: GetterTracker
77
79
onCancel: (listener: (reason: CancelReason) => unknown) => void
78
80
onCleanup: (listener: () => unknown) => void
79
81
providedContext: Record<string, any>
+1
-2
packages/vitest/src/utils/config-helpers.ts
+1
-2
packages/vitest/src/utils/config-helpers.ts
···
1
1
import type {
2
-
BenchmarkBuiltinReporters,
3
2
BuiltinReporters,
4
3
} from '../node/reporters'
5
4
···
9
8
10
9
export function getOutputFile(
11
10
config: PotentialConfig | undefined,
12
-
reporter: BuiltinReporters | BenchmarkBuiltinReporters | 'html',
11
+
reporter: BuiltinReporters | 'html',
13
12
): string | undefined {
14
13
if (!config?.outputFile) {
15
14
return
-32
test/browser/fixtures/benchmark/basic.bench.ts
-32
test/browser/fixtures/benchmark/basic.bench.ts
···
1
-
import { bench, describe } from 'vitest'
2
-
3
-
describe('suite-a', () => {
4
-
bench('good', async () => {
5
-
await sleep(10)
6
-
}, options)
7
-
8
-
bench('bad', async () => {
9
-
await sleep(300)
10
-
}, options)
11
-
})
12
-
13
-
describe('suite-b', () => {
14
-
bench('good', async () => {
15
-
await sleep(25)
16
-
}, options)
17
-
18
-
describe('suite-b-nested', () => {
19
-
bench('good', async () => {
20
-
await sleep(50)
21
-
}, options)
22
-
})
23
-
})
24
-
25
-
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
26
-
27
-
const options = {
28
-
time: 0,
29
-
iterations: 2,
30
-
warmupIterations: 0,
31
-
warmupTime: 0,
32
-
}
-15
test/browser/fixtures/benchmark/vitest.config.ts
-15
test/browser/fixtures/benchmark/vitest.config.ts
···
1
-
import { fileURLToPath } from 'node:url'
2
-
import { defineConfig } from 'vitest/config'
3
-
import { instances, provider } from '../../settings'
4
-
5
-
export default defineConfig({
6
-
test: {
7
-
browser: {
8
-
enabled: true,
9
-
headless: true,
10
-
provider,
11
-
instances,
12
-
},
13
-
},
14
-
cacheDir: fileURLToPath(new URL("./node_modules/.vite", import.meta.url)),
15
-
})
-8
test/e2e/fixtures/mode/example.benchmark.ts
-8
test/e2e/fixtures/mode/example.benchmark.ts
-5
test/e2e/fixtures/mode/example.test.ts
-5
test/e2e/fixtures/mode/example.test.ts
-10
test/e2e/fixtures/mode/vitest.benchmark.config.ts
-10
test/e2e/fixtures/mode/vitest.benchmark.config.ts
-10
test/e2e/fixtures/mode/vitest.test.config.ts
-10
test/e2e/fixtures/mode/vitest.test.config.ts
+10
-4
test/e2e/fixtures/reporters/function-as-name.bench.ts
+10
-4
test/e2e/fixtures/reporters/function-as-name.bench.ts
···
1
-
import { bench } from 'vitest'
1
+
import { test } from 'vitest'
2
2
3
3
const options = {
4
4
time: 0,
···
10
10
function foo() {}
11
11
class Bar {}
12
12
13
-
bench(foo, () => {}, options)
14
-
bench(Bar, () => {}, options)
15
-
bench(() => {}, () => {}, options)
13
+
test('benches', async ({ bench }) => {
14
+
await bench.compare(
15
+
bench(foo, () => {}),
16
+
bench(Bar, () => {}),
17
+
bench(() => {}, () => {}),
18
+
options,
19
+
)
20
+
})
21
+
-33
test/e2e/test/__snapshots__/benchmarking.test.ts.snap
-33
test/e2e/test/__snapshots__/benchmarking.test.ts.snap
···
1
-
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2
-
3
-
exports[`sequential 1`] = `
4
-
"F1 / S1 / B1
5
-
F1 / S1 / B1
6
-
F1 / S1 / B1
7
-
F1 / S1 / B2
8
-
F1 / S1 / B2
9
-
F1 / S1 / B2
10
-
F1 / S2 / B1
11
-
F1 / S2 / B1
12
-
F1 / S2 / B1
13
-
F1 / B1
14
-
F1 / B1
15
-
F1 / B1
16
-
F2 / S1 / B1
17
-
F2 / S1 / B1
18
-
F2 / S1 / B1
19
-
"
20
-
`;
21
-
22
-
exports[`summary 1`] = `
23
-
"
24
-
25
-
good - summary.bench.ts > suite-a
26
-
(?) faster than bad
27
-
28
-
good - summary.bench.ts > suite-b
29
-
30
-
good - summary.bench.ts > suite-b > suite-b-nested
31
-
32
-
"
33
-
`;
+6
-2
test/e2e/test/reporters/function-as-name.test.ts
+6
-2
test/e2e/test/reporters/function-as-name.test.ts
···
15
15
expect(stdout).toContain('function-as-name.test.ts > Bar > Bar')
16
16
})
17
17
18
-
test('should print function name in benchmark', async () => {
18
+
test.for(['default', 'verbose'])('should print function name in benchmark in %s reporter', async (reporters) => {
19
19
const filename = resolve('./fixtures/reporters/function-as-name.bench.ts')
20
-
const { stdout } = await runVitest({ root: './fixtures/reporters' }, [filename], { mode: 'benchmark' })
20
+
const { stdout } = await runVitest({
21
+
root: './fixtures/reporters',
22
+
reporters,
23
+
benchmark: { enabled: true },
24
+
}, [filename])
21
25
22
26
expect(stdout).toBeTruthy()
23
27
expect(stdout).toContain('Bar')
+6
test/e2e/test/reporters/merge-reports.test.ts
+6
test/e2e/test/reporters/merge-reports.test.ts
···
176
176
"assertionResults": [
177
177
{
178
178
"ancestorTitles": [],
179
+
"benchmarks": [],
179
180
"failureMessages": [],
180
181
"fullName": "test 1-1",
181
182
"meta": {},
···
185
186
},
186
187
{
187
188
"ancestorTitles": [],
189
+
"benchmarks": [],
188
190
"failureMessages": [
189
191
"AssertionError: expected 1 to be 2 // Object.is equality
190
192
at <root>/fixtures/reporters/merge-reports/first.test.ts:15:13",
···
206
208
"assertionResults": [
207
209
{
208
210
"ancestorTitles": [],
211
+
"benchmarks": [],
209
212
"failureMessages": [
210
213
"AssertionError: expected 1 to be 2 // Object.is equality
211
214
at <root>/fixtures/reporters/merge-reports/second.test.ts:5:13",
···
220
223
"ancestorTitles": [
221
224
"group",
222
225
],
226
+
"benchmarks": [],
223
227
"failureMessages": [],
224
228
"fullName": "group test 2-2",
225
229
"meta": {},
···
231
235
"ancestorTitles": [
232
236
"group",
233
237
],
238
+
"benchmarks": [],
234
239
"failureMessages": [],
235
240
"fullName": "group test 2-3",
236
241
"meta": {},
···
544
549
result: { state: 'pass' },
545
550
meta: {},
546
551
context: {} as any,
552
+
benchmarks: [],
547
553
}
548
554
}
549
555
+10
test/e2e/test/reporters/utils.ts
+10
test/e2e/test/reporters/utils.ts
···
117
117
duration: 1.4422860145568848,
118
118
},
119
119
context: null as any,
120
+
benchmarks: [],
120
121
})
121
122
122
123
const error: TestError = {
···
165
166
},
166
167
timeout: 0,
167
168
context: null as any,
169
+
benchmarks: [],
168
170
},
169
171
{
170
172
id: `${suite.id}_1`,
···
182
184
file,
183
185
result: { state: 'pass', duration: 1.0237109661102295 },
184
186
context: null as any,
187
+
benchmarks: [],
185
188
},
186
189
{
187
190
id: `${suite.id}_3`,
···
199
202
artifacts: [],
200
203
result: undefined,
201
204
context: null as any,
205
+
benchmarks: [],
202
206
},
203
207
{
204
208
id: `${suite.id}_4`,
···
216
220
file,
217
221
result: { state: 'pass', duration: 100.50598406791687 },
218
222
context: null as any,
223
+
benchmarks: [],
219
224
},
220
225
{
221
226
id: `${suite.id}_5`,
···
233
238
file,
234
239
result: { state: 'pass', duration: 20.184875011444092 },
235
240
context: null as any,
241
+
benchmarks: [],
236
242
},
237
243
{
238
244
id: `${suite.id}_6`,
···
250
256
file,
251
257
result: { state: 'pass', duration: 0.33245420455932617 },
252
258
context: null as any,
259
+
benchmarks: [],
253
260
},
254
261
{
255
262
id: `${suite.id}_7`,
···
267
274
file,
268
275
result: { state: 'pass', duration: 19.738605976104736 },
269
276
context: null as any,
277
+
benchmarks: [],
270
278
},
271
279
{
272
280
id: `${suite.id}_8`,
···
284
292
file,
285
293
result: { state: 'pass', duration: 0.1923508644104004 },
286
294
context: null as any,
295
+
benchmarks: [],
287
296
logs: [
288
297
{
289
298
content: '[33merror[39m',
···
309
318
file,
310
319
result: undefined,
311
320
context: null as any,
321
+
benchmarks: [],
312
322
},
313
323
]
314
324
+17
-22
packages/browser/src/client/tester/runner.ts
+17
-22
packages/browser/src/client/tester/runner.ts
···
8
8
Test,
9
9
TestAnnotation,
10
10
TestArtifact,
11
+
TestTryOptions,
11
12
VitestRunner,
12
13
} from '@vitest/runner'
13
14
import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest'
14
-
import type { Traces } from 'vitest/internal/traces'
15
15
import type { VitestBrowserClientMocker } from './mocker'
16
16
import type { CommandsManager } from './tester-utils'
17
17
import { globalChannel, onCancel } from '@vitest/browser/client'
18
18
import { getTestName } from '@vitest/runner/utils'
19
-
import { BenchmarkRunner, recordArtifact, TestRunner } from 'vitest'
19
+
import { recordArtifact, TestRunner } from 'vitest'
20
20
import { page, userEvent } from 'vitest/browser'
21
21
import {
22
22
DecodedMap,
···
48
48
}
49
49
50
50
export function createBrowserRunner(
51
-
runnerClass: { new (config: SerializedConfig): VitestRunner },
52
51
mocker: VitestBrowserClientMocker,
53
52
state: WorkerGlobalState,
54
-
coverageModule: CoverageHandler | null,
53
+
coverageModule: CoverageHandler,
55
54
): { new (options: BrowserRunnerOptions): BrowserVitestRunner } {
56
-
return class BrowserTestRunner extends runnerClass implements VitestRunner {
55
+
return class BrowserTestRunner extends TestRunner implements VitestRunner {
57
56
public config: SerializedConfig
58
-
hashMap = browserHashMap
57
+
public hashMap = browserHashMap
59
58
public sourceMapCache = new Map<string, any>()
60
59
public method = 'run' as TestExecutionMethod
61
60
private commands: CommandsManager
62
-
private _otel!: Traces
63
61
64
62
constructor(options: BrowserRunnerOptions) {
65
63
super(options.config)
···
75
73
76
74
private traces = new Map<string, string[]>()
77
75
78
-
onBeforeTryTask: VitestRunner['onBeforeTryTask'] = async (...args) => {
76
+
async onBeforeTryTask(test: Test, options: TestTryOptions) {
79
77
await userEvent.cleanup()
80
-
await super.onBeforeTryTask?.(...args)
78
+
super.onBeforeTryTask?.(test, options)
81
79
const trace = this.config.browser.trace
82
-
const test = args[0]
83
-
const { retry, repeats } = args[1]
80
+
const { retry, repeats } = options
84
81
const shouldTrace = trace !== 'off'
85
82
&& !(trace === 'on-all-retries' && retry === 0)
86
83
&& !(trace === 'on-first-retry' && retry !== 1)
···
152
149
}
153
150
154
151
onAfterRunTask = async (task: Test) => {
155
-
await super.onAfterRunTask?.(task)
152
+
super.onAfterRunTask?.(task)
156
153
const trace = this.config.browser.trace
157
154
const traces = this.traces.get(task.id) || []
158
155
if (traces.length) {
···
230
227
}
231
228
232
229
onAfterRunFiles = async (files: File[]) => {
230
+
super.onAfterRunFiles(files)
231
+
233
232
const [coverage] = await Promise.all([
234
-
coverageModule?.takeCoverage?.(),
233
+
coverageModule.takeCoverage(),
235
234
mocker.invalidate(),
236
-
super.onAfterRunFiles?.(files),
237
235
])
238
236
239
237
if (coverage) {
···
357
355
if (cachedRunner) {
358
356
return cachedRunner
359
357
}
360
-
const runnerClass
361
-
= config.mode === 'test' ? TestRunner : BenchmarkRunner
362
-
363
-
const BrowserRunner = createBrowserRunner(runnerClass, mocker, state, {
358
+
const BrowserRunner = createBrowserRunner(mocker, state, {
364
359
takeCoverage: () =>
365
360
takeCoverageInsideWorker(config.coverage, moduleRunner),
366
361
})
···
377
372
})
378
373
379
374
const [diffOptions] = await Promise.all([
380
-
loadDiffConfig(config, moduleRunner as any),
381
-
loadSnapshotSerializers(config, moduleRunner as any),
375
+
loadDiffConfig(config, moduleRunner),
376
+
loadSnapshotSerializers(config, moduleRunner),
382
377
])
383
-
runner.config.diffOptions = diffOptions
378
+
runner.config._diffOptions = diffOptions
384
379
getWorkerState().onFilterStackTrace = (stack: string) => {
385
380
const stacks = parseStacktrace(stack, {
386
381
getSourceMap(file) {
···
400
395
if (!result) {
401
396
return null
402
397
}
403
-
return new DecodedMap(result as any, file)
398
+
return new DecodedMap(result, file)
404
399
}
405
400
406
401
async function updateTestFilesLocations(files: File[], sourceMaps: Map<string, any>) {
+83
packages/vitest/src/integrations/chai/bench.ts
+83
packages/vitest/src/integrations/chai/bench.ts
···
1
+
import type { MatchersObject } from '@vitest/expect'
2
+
import type { BenchResult } from '../../runtime/benchmark'
3
+
4
+
function isBenchResult(value: unknown): value is BenchResult {
5
+
return (
6
+
typeof value === 'object'
7
+
&& value !== null
8
+
&& 'latency' in value
9
+
&& typeof (value as any).latency?.mean === 'number'
10
+
)
11
+
}
12
+
13
+
function formatOps(ops: number): string {
14
+
return ops.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })
15
+
}
16
+
17
+
export const benchMatchers: MatchersObject = {
18
+
toBeFasterThan(actual: unknown, expected: unknown, options?: { delta?: number }) {
19
+
const { matcherHint, RECEIVED_COLOR, EXPECTED_COLOR } = this.utils
20
+
const delta = options?.delta ?? 0
21
+
22
+
if (!isBenchResult(actual)) {
23
+
throw new TypeError(
24
+
`${matcherHint('.toBeFasterThan')} expects the actual value to be a benchmark result.`,
25
+
)
26
+
}
27
+
if (!isBenchResult(expected)) {
28
+
throw new TypeError(
29
+
`${matcherHint('.toBeFasterThan')} expects the expected value to be a benchmark result.`,
30
+
)
31
+
}
32
+
33
+
const threshold = expected.latency.mean * (1 - delta)
34
+
const pass = actual.latency.mean < threshold
35
+
36
+
return {
37
+
pass,
38
+
message: () => {
39
+
const relation = ((actual.latency.mean - expected.latency.mean) / expected.latency.mean * 100).toFixed(2)
40
+
return pass
41
+
? `${matcherHint('.not.toBeFasterThan')}\n\nExpected to not be faster, but was ${Math.abs(Number(relation))}% faster.\n\n`
42
+
+ `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
43
+
+ `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
44
+
: `${matcherHint('.toBeFasterThan')}\n\nExpected to be faster${delta > 0 ? ` by at least ${(delta * 100).toFixed(0)}%` : ''}, but was ${Number(relation) > 0 ? `${relation}% slower` : `only ${Math.abs(Number(relation))}% faster`}.\n\n`
45
+
+ `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
46
+
+ `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
47
+
},
48
+
}
49
+
},
50
+
51
+
toBeSlowerThan(actual: unknown, expected: unknown, options?: { delta?: number }) {
52
+
const { matcherHint, RECEIVED_COLOR, EXPECTED_COLOR } = this.utils
53
+
const delta = options?.delta ?? 0
54
+
55
+
if (!isBenchResult(actual)) {
56
+
throw new TypeError(
57
+
`${matcherHint('.toBeSlowerThan')} expects the actual value to be a benchmark result.`,
58
+
)
59
+
}
60
+
if (!isBenchResult(expected)) {
61
+
throw new TypeError(
62
+
`${matcherHint('.toBeSlowerThan')} expects the expected value to be a benchmark result.`,
63
+
)
64
+
}
65
+
66
+
const threshold = expected.latency.mean * (1 + delta)
67
+
const pass = actual.latency.mean > threshold
68
+
69
+
return {
70
+
pass,
71
+
message: () => {
72
+
const relation = ((actual.latency.mean - expected.latency.mean) / expected.latency.mean * 100).toFixed(2)
73
+
return pass
74
+
? `${matcherHint('.not.toBeSlowerThan')}\n\nExpected to not be slower, but was ${relation}% slower.\n\n`
75
+
+ `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
76
+
+ `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
77
+
: `${matcherHint('.toBeSlowerThan')}\n\nExpected to be slower${delta > 0 ? ` by at least ${(delta * 100).toFixed(0)}%` : ''}, but was ${Number(relation) < 0 ? `${Math.abs(Number(relation))}% faster` : `only ${relation}% slower`}.\n\n`
78
+
+ `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
79
+
+ `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
80
+
},
81
+
}
82
+
},
83
+
}
+2
packages/vitest/src/integrations/chai/index.ts
+2
packages/vitest/src/integrations/chai/index.ts
···
11
11
} from '@vitest/expect'
12
12
import { getCurrentTest } from '@vitest/runner'
13
13
import { getWorkerState } from '../../runtime/utils'
14
+
import { benchMatchers } from './bench'
14
15
import { createExpectPoll } from './poll'
15
16
import './setup'
16
17
···
108
109
chai.util.addMethod(expect, 'hasAssertions', hasAssertions)
109
110
110
111
expect.extend(customMatchers)
112
+
expect.extend(benchMatchers)
111
113
112
114
return expect
113
115
}
+18
-20
packages/vitest/src/node/cli/cac.ts
+18
-20
packages/vitest/src/node/cli/cac.ts
···
1
1
import type { CAC, Command } from 'cac'
2
-
import type { VitestRunMode } from '../types/config'
3
2
import type { CliOptions } from './cli-api'
4
3
import type { CLIOption, CLIOptions as CLIOptionsConfig } from './cli-config'
5
4
import { toArray } from '@vitest/utils/helpers'
6
5
import cac from 'cac'
7
6
import { normalize } from 'pathe'
8
-
import c, { disableDefaultColors } from 'tinyrainbow'
7
+
import { disableDefaultColors } from 'tinyrainbow'
9
8
import { version } from '../../../package.json' with { type: 'json' }
10
9
import { isAgent, isForceColor } from '../../utils/env'
11
-
import { benchCliOptionsConfig, cliOptionsConfig, collectCliOptionsConfig } from './cli-config'
10
+
import { cliOptionsConfig, collectCliOptionsConfig } from './cli-config'
12
11
import { setupTabCompletions } from './completions'
13
12
14
13
function addCommand(cli: CAC | Command, name: string, option: CLIOption<any>) {
···
179
178
.command('dev [...filters]', undefined, options)
180
179
.action(watch)
181
180
182
-
addCliOptions(
183
-
cli
184
-
.command('bench [...filters]', undefined, options)
185
-
.action(benchmark),
186
-
benchCliOptionsConfig,
187
-
)
181
+
cli
182
+
.command('bench [...filters]', undefined, options)
183
+
.action(benchmark)
188
184
189
185
cli
190
186
.command('init <project>', undefined, options)
···
193
189
addCliOptions(
194
190
cli
195
191
.command('list [...filters]', undefined, options)
196
-
.action((filters, options) => collect('test', filters, options)),
192
+
.action((filters, options) => collect(filters, options)),
197
193
collectCliOptionsConfig,
198
194
)
199
195
200
196
cli
201
197
.command('[...filters]', undefined, options)
202
-
.action((filters, options) => start('test', filters, options))
198
+
.action((filters, options) => start(filters, options))
203
199
204
200
setupTabCompletions(cli)
205
201
return cli
···
263
259
async function runRelated(relatedFiles: string[] | string, argv: CliOptions): Promise<void> {
264
260
argv.related = relatedFiles
265
261
argv.passWithNoTests ??= true
266
-
await start('test', [], argv)
262
+
await start([], argv)
267
263
}
268
264
269
265
async function watch(cliFilters: string[], options: CliOptions): Promise<void> {
270
266
options.watch = true
271
-
await start('test', cliFilters, options)
267
+
await start(cliFilters, options)
272
268
}
273
269
274
270
async function run(cliFilters: string[], options: CliOptions): Promise<void> {
275
271
// "vitest run --watch" should still be watch mode
276
272
options.run = !options.watch
277
273
278
-
await start('test', cliFilters, options)
274
+
await start(cliFilters, options)
279
275
}
280
276
281
277
async function benchmark(cliFilters: string[], options: CliOptions): Promise<void> {
282
-
console.warn(c.yellow('Benchmarking is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.'))
283
-
await start('benchmark', cliFilters, options)
278
+
options.benchmarkOnly = true
279
+
options.coverage ??= {}
280
+
options.coverage.enabled = false
281
+
await start(cliFilters, options)
284
282
}
285
283
286
284
function normalizeCliOptions(cliFilters: string[], argv: CliOptions): CliOptions {
···
303
301
return argv
304
302
}
305
303
306
-
async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<void> {
304
+
async function start(cliFilters: string[], options: CliOptions): Promise<void> {
307
305
try {
308
306
const { startVitest } = await import('./cli-api')
309
-
const ctx = await startVitest(mode, cliFilters.map(normalize), normalizeCliOptions(cliFilters, options))
307
+
const ctx = await startVitest(cliFilters.map(normalize), normalizeCliOptions(cliFilters, options))
310
308
if (!ctx.shouldKeepServer()) {
311
309
await ctx.exit()
312
310
}
···
335
333
await create()
336
334
}
337
335
338
-
async function collect(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<void> {
336
+
async function collect(cliFilters: string[], options: CliOptions): Promise<void> {
339
337
try {
340
338
const { prepareVitest, processCollected, outputFileList } = await import('./cli-api')
341
-
const ctx = await prepareVitest(mode, {
339
+
const ctx = await prepareVitest({
342
340
...normalizeCliOptions(cliFilters, options),
343
341
watch: false,
344
342
run: true,
+78
-8
packages/vitest/src/node/cli/cli-api.ts
+78
-8
packages/vitest/src/node/cli/cli-api.ts
···
46
46
* @experimental
47
47
*/
48
48
configLoader?: ViteInlineConfig extends { configLoader?: infer T } ? T : never
49
+
50
+
/**
51
+
* Only run benchmark projects, filtering out all other projects.
52
+
* Set automatically by `vitest bench`.
53
+
* @internal
54
+
*/
55
+
benchmarkOnly?: boolean
49
56
}
50
57
51
58
/**
···
54
61
* Returns a Vitest instance if initialized successfully.
55
62
*/
56
63
export async function startVitest(
57
-
mode: VitestRunMode,
58
-
cliFilters: string[] = [],
59
-
options: CliOptions = {},
64
+
cliFilters?: string[],
65
+
options?: CliOptions,
60
66
viteOverrides?: ViteUserConfig,
61
67
vitestOptions?: VitestOptions,
68
+
): Promise<Vitest>
69
+
/**
70
+
* @deprecated The `mode` argument is no longer used. Use `startVitest(cliFilters?, options?, viteOverrides?, vitestOptions?)` instead.
71
+
*/
72
+
export async function startVitest(
73
+
mode: VitestRunMode,
74
+
cliFilters?: string[],
75
+
options?: CliOptions,
76
+
viteOverrides?: ViteUserConfig,
77
+
vitestOptions?: VitestOptions,
78
+
): Promise<Vitest>
79
+
export async function startVitest(
80
+
modeOrCliFilters?: VitestRunMode | string[],
81
+
cliFiltersOrOptions?: string[] | CliOptions,
82
+
optionsOrViteOverrides?: CliOptions | ViteUserConfig,
83
+
viteOverridesOrVitestOptions?: ViteUserConfig | VitestOptions,
84
+
maybeVitestOptions?: VitestOptions,
62
85
): Promise<Vitest> {
86
+
let cliFilters: string[]
87
+
let options: CliOptions
88
+
let viteOverrides: ViteUserConfig | undefined
89
+
let vitestOptions: VitestOptions | undefined
90
+
if (typeof modeOrCliFilters === 'string') {
91
+
cliFilters = (cliFiltersOrOptions as string[] | undefined) ?? []
92
+
options = (optionsOrViteOverrides as CliOptions | undefined) ?? {}
93
+
viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig | undefined
94
+
vitestOptions = maybeVitestOptions
95
+
}
96
+
else {
97
+
cliFilters = modeOrCliFilters ?? []
98
+
options = (cliFiltersOrOptions as CliOptions | undefined) ?? {}
99
+
viteOverrides = optionsOrViteOverrides as ViteUserConfig | undefined
100
+
vitestOptions = viteOverridesOrVitestOptions as VitestOptions | undefined
101
+
}
63
102
const root = resolve(options.root || process.cwd())
64
103
65
104
const ctx = await prepareVitest(
66
-
mode,
67
105
options,
68
106
viteOverrides,
69
107
vitestOptions,
70
108
cliFilters,
71
109
)
72
110
73
-
if (mode === 'test' && ctx._coverageOptions.enabled) {
111
+
if (ctx._coverageOptions.enabled) {
74
112
const provider = ctx._coverageOptions.provider || 'v8'
75
113
const requiredPackages = CoverageProviderMap[provider]
76
114
···
151
189
}
152
190
153
191
export async function prepareVitest(
154
-
mode: VitestRunMode,
155
-
options: CliOptions = {},
192
+
options?: CliOptions,
156
193
viteOverrides?: ViteUserConfig,
157
194
vitestOptions?: VitestOptions,
158
195
cliFilters?: string[],
196
+
): Promise<Vitest>
197
+
/**
198
+
* @deprecated The `mode` argument is no longer used. Use `prepareVitest(options?, viteOverrides?, vitestOptions?, cliFilters?)` instead.
199
+
*/
200
+
export async function prepareVitest(
201
+
mode: VitestRunMode,
202
+
options?: CliOptions,
203
+
viteOverrides?: ViteUserConfig,
204
+
vitestOptions?: VitestOptions,
205
+
cliFilters?: string[],
206
+
): Promise<Vitest>
207
+
export async function prepareVitest(
208
+
modeOrOptions?: VitestRunMode | CliOptions,
209
+
optionsOrViteOverrides?: CliOptions | ViteUserConfig,
210
+
viteOverridesOrVitestOptions?: ViteUserConfig | VitestOptions,
211
+
vitestOptionsOrCliFilters?: VitestOptions | string[],
212
+
maybeCliFilters?: string[],
159
213
): Promise<Vitest> {
214
+
let options: CliOptions
215
+
let viteOverrides: ViteUserConfig | undefined
216
+
let vitestOptions: VitestOptions | undefined
217
+
let cliFilters: string[] | undefined
218
+
if (typeof modeOrOptions === 'string') {
219
+
options = (optionsOrViteOverrides as CliOptions | undefined) ?? {}
220
+
viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig | undefined
221
+
vitestOptions = vitestOptionsOrCliFilters as VitestOptions | undefined
222
+
cliFilters = maybeCliFilters
223
+
}
224
+
else {
225
+
options = modeOrOptions ?? {}
226
+
viteOverrides = optionsOrViteOverrides as ViteUserConfig | undefined
227
+
vitestOptions = viteOverridesOrVitestOptions as VitestOptions | undefined
228
+
cliFilters = vitestOptionsOrCliFilters as string[] | undefined
229
+
}
160
230
process.env.TEST = 'true'
161
231
process.env.VITEST = 'true'
162
232
process.env.NODE_ENV ??= 'test'
···
172
242
// this shouldn't affect _application root_ that can be changed inside config
173
243
const root = resolve(options.root || process.cwd())
174
244
175
-
const ctx = await createVitest(mode, options, viteOverrides, vitestOptions)
245
+
const ctx = await createVitest(options, viteOverrides, vitestOptions)
176
246
177
247
const environmentPackage = getEnvPackageName(ctx.config.environment)
178
248
+2
-17
packages/vitest/src/node/cli/cli-config.ts
+2
-17
packages/vitest/src/node/cli/cli-config.ts
···
332
332
},
333
333
},
334
334
mode: {
335
-
description: 'Override Vite mode (default: `test` or `benchmark`)',
335
+
description: 'Override Vite mode (default: `test`)',
336
336
argument: '<name>',
337
337
},
338
338
isolate: {
···
976
976
deps: null,
977
977
name: null,
978
978
snapshotEnvironment: null,
979
-
compare: null,
980
-
outputJson: null,
981
979
json: null,
982
980
provide: null,
983
981
filesOnly: null,
···
986
984
projects: null,
987
985
watchTriggerPatterns: null,
988
986
tags: null,
987
+
benchmarkOnly: null,
989
988
taskTitleValueFormatTruncate: null,
990
-
}
991
-
992
-
export const benchCliOptionsConfig: Pick<
993
-
VitestCLIOptions,
994
-
'compare' | 'outputJson'
995
-
> = {
996
-
compare: {
997
-
description: 'Benchmark output file to compare against',
998
-
argument: '<filename>',
999
-
},
1000
-
outputJson: {
1001
-
description: 'Benchmark output file',
1002
-
argument: '<filename>',
1003
-
},
1004
989
}
1005
990
1006
991
export const collectCliOptionsConfig: VitestCLIOptions = {
+34
-71
packages/vitest/src/node/config/resolveConfig.ts
+34
-71
packages/vitest/src/node/config/resolveConfig.ts
···
1
1
import type { ResolvedConfig as ResolvedViteConfig } from 'vite'
2
2
import type { Vitest } from '../core'
3
3
import type { Logger } from '../logger'
4
-
import type { BenchmarkBuiltinReporters } from '../reporters'
5
4
import type { ResolvedBrowserOptions } from '../types/browser'
6
5
import type {
7
6
ApiConfig,
···
155
154
options: UserConfig,
156
155
viteConfig: ResolvedViteConfig,
157
156
): ResolvedConfig {
158
-
const mode = vitest.mode
159
157
const logger = vitest.logger
160
158
if (options.dom) {
161
159
if (
···
178
176
...configDefaults,
179
177
...options,
180
178
root: viteConfig.root,
181
-
mode,
182
179
} as any as ResolvedConfig
180
+
181
+
resolved.mode ??= viteConfig.mode ?? 'test'
183
182
184
183
if (resolved.retry && typeof resolved.retry === 'object' && typeof resolved.retry.condition === 'function') {
185
184
logger.console.warn(
···
245
244
throw new Error(`Looks like you set "test.environment" to "browser". To enable Browser Mode, use "test.browser.enabled" instead.`)
246
245
}
247
246
247
+
resolved.benchmark = {
248
+
...benchmarkConfigDefaults,
249
+
...resolved.benchmark,
250
+
}
251
+
248
252
const inspector = resolved.inspect || resolved.inspectBrk
249
253
250
254
resolved.inspector = {
···
295
299
resolved.maxWorkers = resolveInlineWorkerOption(resolved.maxWorkers)
296
300
}
297
301
298
-
// run benchmark sequentially by default
299
-
const fileParallelism = options.fileParallelism ?? mode !== 'benchmark'
302
+
const fileParallelism = options.fileParallelism ?? true
300
303
301
304
if (!fileParallelism) {
302
305
// ignore user config, parallelism cannot be implemented without limiting workers
···
634
637
resolved.maxWorkers = Number.parseInt(process.env.VITEST_MAX_WORKERS)
635
638
}
636
639
637
-
if (mode === 'benchmark') {
638
-
resolved.benchmark = {
639
-
...benchmarkConfigDefaults,
640
-
...resolved.benchmark,
641
-
}
642
-
// override test config
643
-
resolved.coverage.enabled = false
644
-
resolved.typecheck.enabled = false
645
-
resolved.include = resolved.benchmark.include
646
-
resolved.exclude = resolved.benchmark.exclude
647
-
resolved.includeSource = resolved.benchmark.includeSource
648
-
const reporters = Array.from(
649
-
new Set<BenchmarkBuiltinReporters>([
650
-
...toArray(resolved.benchmark.reporters),
651
-
// @ts-expect-error reporter is CLI flag
652
-
...toArray(options.reporter),
653
-
]),
654
-
).filter(Boolean)
655
-
if (reporters.length) {
656
-
resolved.benchmark.reporters = reporters
657
-
}
658
-
else {
659
-
resolved.benchmark.reporters = ['default']
660
-
}
661
-
662
-
if (options.outputFile) {
663
-
resolved.benchmark.outputFile = options.outputFile
664
-
}
665
-
666
-
// --compare from cli
667
-
if (options.compare) {
668
-
resolved.benchmark.compare = options.compare
669
-
}
670
-
if (options.outputJson) {
671
-
resolved.benchmark.outputJson = options.outputJson
672
-
}
673
-
}
674
-
675
640
if (typeof resolved.diff === 'string') {
676
641
resolved.diff = resolvePath(resolved.diff, resolved.root)
677
642
resolved.forceRerunTriggers.push(resolved.diff)
···
729
694
}
730
695
}
731
696
732
-
if (mode !== 'benchmark') {
733
-
// @ts-expect-error "reporter" is from CLI, should be absolute to the running directory
734
-
// it is passed down as "vitest --reporter ../reporter.js"
735
-
const reportersFromCLI = resolved.reporter
697
+
// @ts-expect-error "reporter" is from CLI, should be absolute to the running directory
698
+
// it is passed down as "vitest --reporter ../reporter.js"
699
+
const reportersFromCLI = resolved.reporter
736
700
737
-
const cliReporters = toArray(reportersFromCLI || []).map(
738
-
(reporter: string) => {
739
-
// ./reporter.js || ../reporter.js, but not .reporters/reporter.js
740
-
if (/^\.\.?\//.test(reporter)) {
741
-
return resolve(process.cwd(), reporter)
742
-
}
743
-
return reporter
744
-
},
745
-
)
701
+
const cliReporters = toArray(reportersFromCLI || []).map(
702
+
(reporter: string) => {
703
+
// ./reporter.js || ../reporter.js, but not .reporters/reporter.js
704
+
if (/^\.\.?\//.test(reporter)) {
705
+
return resolve(process.cwd(), reporter)
706
+
}
707
+
return reporter
708
+
},
709
+
)
746
710
747
-
if (cliReporters.length) {
748
-
// When CLI reporters are specified, preserve options from config file
749
-
const configReportersMap = new Map<string, Record<string, unknown>>()
711
+
if (cliReporters.length) {
712
+
// When CLI reporters are specified, preserve options from config file
713
+
const configReportersMap = new Map<string, Record<string, unknown>>()
750
714
751
-
// Build a map of reporter names to their options from the config
752
-
for (const reporter of resolved.reporters) {
753
-
if (Array.isArray(reporter)) {
754
-
const [reporterName, reporterOptions] = reporter
755
-
if (typeof reporterName === 'string') {
756
-
configReportersMap.set(reporterName, reporterOptions as Record<string, unknown>)
757
-
}
715
+
// Build a map of reporter names to their options from the config
716
+
for (const reporter of resolved.reporters) {
717
+
if (Array.isArray(reporter)) {
718
+
const [reporterName, reporterOptions] = reporter
719
+
if (typeof reporterName === 'string') {
720
+
configReportersMap.set(reporterName, reporterOptions as Record<string, unknown>)
758
721
}
759
722
}
760
-
761
-
resolved.reporters = Array.from(new Set(toArray(cliReporters)))
762
-
.filter(Boolean)
763
-
.map(reporter => [reporter, configReportersMap.get(reporter) || {}])
764
723
}
724
+
725
+
resolved.reporters = Array.from(new Set(toArray(cliReporters)))
726
+
.filter(Boolean)
727
+
.map(reporter => [reporter, configReportersMap.get(reporter) || {}])
765
728
}
766
729
767
730
resolved.mergeReportsLabel = process.env.VITEST_BLOB_LABEL
···
842
805
}
843
806
resolved.browser.isolate ??= resolved.isolate ?? true
844
807
resolved.browser.fileParallelism
845
-
??= options.fileParallelism ?? mode !== 'benchmark'
808
+
??= options.fileParallelism ?? true
846
809
// disable in headless mode by default, and if CI is detected
847
810
resolved.browser.ui ??= resolved.browser.headless === true ? false : !isCI
848
811
resolved.browser.commands ??= {}
+5
-3
packages/vitest/src/node/config/serializeConfig.ts
+5
-3
packages/vitest/src/node/config/serializeConfig.ts
···
13
13
return {
14
14
// TODO: remove functions from environmentOptions
15
15
environmentOptions: config.environmentOptions,
16
-
mode: config.mode,
17
16
isolate: config.isolate,
18
17
maxWorkers: config.maxWorkers,
19
18
base: config.base,
···
134
133
standalone: config.standalone,
135
134
printConsoleTrace:
136
135
config.printConsoleTrace ?? globalConfig.printConsoleTrace,
137
-
benchmark: config.benchmark && {
138
-
includeSamples: config.benchmark.includeSamples,
136
+
benchmark: {
137
+
enabled: config.benchmark.enabled,
138
+
retainSamples: config.benchmark.retainSamples,
139
+
suppressExportGetterWarnings: config.benchmark.suppressExportGetterWarnings,
140
+
projectName: config.benchmark.projectName,
139
141
},
140
142
// the browser initialized them via `@vite/env` import
141
143
serializedDefines: config.browser.enabled
+6
-1
packages/vitest/src/node/plugins/index.ts
+6
-1
packages/vitest/src/node/plugins/index.ts
···
25
25
26
26
export async function VitestPlugin(
27
27
options: UserConfig = {},
28
-
vitest: Vitest = new Vitest('test', deepClone(options)),
28
+
vitest: Vitest = new Vitest(deepClone(options)),
29
29
): Promise<VitePlugin[]> {
30
30
const userConfig = deepMerge({}, options) as UserConfig
31
31
···
148
148
legalComments: 'inline',
149
149
},
150
150
}
151
+
}
152
+
153
+
if (vitest._cliOptions.benchmarkOnly) {
154
+
config.test!.benchmark ??= {}
155
+
config.test!.benchmark.enabled = true
151
156
}
152
157
153
158
// inherit so it's available in VitestOptimizer
+1
-2
packages/vitest/src/node/plugins/publicConfig.ts
+1
-2
packages/vitest/src/node/plugins/publicConfig.ts
···
27
27
: find.any(configFiles, { cwd: root })
28
28
options.config = configPath
29
29
30
-
const vitest = new Vitest('test', deepClone(options))
30
+
const vitest = new Vitest(deepClone(options))
31
31
const config = await resolveViteConfig(
32
32
mergeConfig(
33
33
{
34
34
configFile: configPath,
35
-
// this will make "mode": "test" | "benchmark" inside defineConfig
36
35
mode: options.mode || 'test',
37
36
plugins: [
38
37
await VitestPlugin(options, vitest),
+11
-1
packages/vitest/src/node/plugins/workspace.ts
+11
-1
packages/vitest/src/node/plugins/workspace.ts
···
37
37
name: 'vitest:project:name',
38
38
enforce: 'post',
39
39
config(viteConfig) {
40
-
const testConfig = viteConfig.test || {}
40
+
viteConfig.test ??= {}
41
+
42
+
const testConfig = viteConfig.test
41
43
42
44
let { label: name, color } = typeof testConfig.name === 'string'
43
45
? { label: testConfig.name }
···
62
64
}
63
65
}
64
66
67
+
if (project.vitest._cliOptions.benchmarkOnly) {
68
+
viteConfig.test.benchmark ??= {}
69
+
viteConfig.test.benchmark.enabled = true
70
+
}
71
+
65
72
const isUserBrowserEnabled = viteConfig.test?.browser?.enabled
66
73
const isBrowserEnabled = isUserBrowserEnabled ?? (viteConfig.test?.browser && project.vitest._cliOptions.browser?.enabled)
67
74
// keep project names to potentially filter it out
···
79
86
workspaceNames.push(instance.name)
80
87
}
81
88
})
89
+
if (viteConfig.test?.benchmark?.enabled) {
90
+
workspaceNames.push(name ? `${name} (bench)` : 'bench')
91
+
}
82
92
83
93
const filters = project.vitest.config.project
84
94
// if there is `--project=...` filter, check if any of the potential projects match
+1
-1
packages/vitest/src/node/pools/poolRunner.ts
+1
-1
packages/vitest/src/node/pools/poolRunner.ts
+9
packages/vitest/src/node/pools/rpc.ts
+9
packages/vitest/src/node/pools/rpc.ts
···
121
121
onAfterSuiteRun(meta) {
122
122
vitest.coverageProvider?.onAfterSuiteRun(meta)
123
123
},
124
+
async onTestBenchmark(testId, benchmark) {
125
+
return vitest._testRun.recordBenchmark(testId, benchmark)
126
+
},
127
+
async readBenchmarkResult(relativePath) {
128
+
return project.benchmark.readResult(relativePath)
129
+
},
130
+
async writeBenchmarkResult(relativePath, data) {
131
+
return project.benchmark.writeResult(relativePath, data)
132
+
},
124
133
async onTaskArtifactRecord(testId, artifact) {
125
134
return vitest._testRun.recordArtifact(testId, artifact)
126
135
},
+77
-3
packages/vitest/src/node/projects/resolveProjects.ts
+77
-3
packages/vitest/src/node/projects/resolveProjects.ts
···
197
197
names.add(name)
198
198
}
199
199
200
-
return resolveBrowserProjects(vitest, names, resolvedProjects)
200
+
return resolveDefaultProjects(vitest, names, resolvedProjects)
201
201
}
202
202
203
-
export async function resolveBrowserProjects(
203
+
export async function resolveDefaultProjects(
204
+
vitest: Vitest,
205
+
names: Set<string>,
206
+
resolvedProjects: TestProject[],
207
+
): Promise<TestProject[]> {
208
+
const newProjects = await resolveBrowserProjects(vitest, names, resolvedProjects)
209
+
210
+
let lastGroupOrder = Math.max(0, ...newProjects.map(p => p.config.sequence.groupOrder))
211
+
212
+
newProjects.forEach((project) => {
213
+
const benchmark = project.config.benchmark
214
+
if (!benchmark.enabled) {
215
+
return
216
+
}
217
+
218
+
if (vitest.isExcludedByProjectFilter(project.config.name)) {
219
+
benchmark.enabled = false
220
+
return
221
+
}
222
+
223
+
const name = project.config.name ? `${project.config.name} (bench)` : 'bench'
224
+
if (!vitest.matchesProjectFilter(name)) {
225
+
benchmark.enabled = false
226
+
return
227
+
}
228
+
229
+
if (names.has(name)) {
230
+
throw new Error(`Cannot create a benchmark project because the name "${name}" is already in use.`)
231
+
}
232
+
names.add(name)
233
+
234
+
const benchmarkProject = TestProject._cloneTestProject(project, {
235
+
...project.config,
236
+
name,
237
+
include: benchmark.include,
238
+
exclude: benchmark.exclude,
239
+
includeSource: benchmark.includeSource,
240
+
coverage: {
241
+
...project.config.coverage,
242
+
enabled: false,
243
+
},
244
+
maxWorkers: 1,
245
+
maxConcurrency: 1,
246
+
testTimeout: project.config.testTimeout < 60_000 ? 60_000 : project.config.testTimeout,
247
+
hookTimeout: project.config.hookTimeout < 120_000 ? 120_000 : project.config.hookTimeout,
248
+
// Spread because we disable it in the original project. `projectName`
249
+
// carries the parent's name so the runtime can substitute it into
250
+
// `${projectName}` placeholders inside `writeResult` / `bench.from()`
251
+
// paths — `project.config.name` already excludes the ` (bench)` suffix
252
+
// we add to the cloned project's own name above.
253
+
benchmark: { ...benchmark, projectName: project.config.name ?? '' },
254
+
sequence: {
255
+
...project.config.sequence,
256
+
concurrent: false,
257
+
// benchmarks should always run in a separate isolated group
258
+
groupOrder: ++lastGroupOrder,
259
+
},
260
+
typecheck: {
261
+
...project.config.typecheck,
262
+
enabled: false,
263
+
},
264
+
// TODO: mark if benchmark project?
265
+
})
266
+
// disable benchmark in the original project
267
+
benchmark.enabled = false
268
+
newProjects.push(benchmarkProject)
269
+
})
270
+
return newProjects
271
+
}
272
+
273
+
async function resolveBrowserProjects(
204
274
vitest: Vitest,
205
275
names: Set<string>,
206
276
resolvedProjects: TestProject[],
···
260
330
names.add(name)
261
331
const clonedConfig = cloneConfig(project, config)
262
332
clonedConfig.name = name
263
-
const clone = TestProject._cloneBrowserProject(project, clonedConfig)
333
+
const clone = TestProject._cloneTestProject(project, clonedConfig)
264
334
resolvedProjects.push(clone)
265
335
})
266
336
···
481
551
482
552
function getPotentialProjectNames(project: TestProject) {
483
553
const names = [project.name]
554
+
// TODO: include benchmarks in browsers
484
555
if (project.config.browser.instances) {
485
556
names.push(...project.config.browser.instances.map(i => i.name!))
486
557
}
487
558
else if (project.config.browser.name) {
488
559
names.push(project.config.browser.name)
560
+
}
561
+
if (project.config.benchmark.enabled) {
562
+
names.push(project.name ? `${project.name} (bench)` : 'bench')
489
563
}
490
564
return names
491
565
}
+107
-27
packages/vitest/src/node/reporters/base.ts
+107
-27
packages/vitest/src/node/reporters/base.ts
···
1
-
import type { File, Task, TestAnnotation } from '@vitest/runner'
1
+
import type { File, Task, TestAnnotation, TestBenchmark, TestBenchmarkTask } from '@vitest/runner'
2
2
import type { ParsedStack, SerializedError } from '@vitest/utils'
3
3
import type { AsyncLeak, TestError, UserConsoleLog } from '../../types/general'
4
4
import type { Vitest } from '../core'
···
16
16
import { isTTY } from '../../utils/env'
17
17
import { hasFailedSnapshot } from '../../utils/tasks'
18
18
import { generateCodeFrame, printStack } from '../printError'
19
+
import { BENCH_TABLE_HEAD, computeBenchColumnWidths, padBenchRow, renderBenchmarkRow } from './renderers/benchmark-table'
19
20
import { F_CHECK, F_DOWN_RIGHT, F_POINTER } from './renderers/figures'
20
21
import {
21
22
countTestErrors,
···
54
55
55
56
private _filesInWatchMode = new Map<string, number>()
56
57
private _timeStart = formatTimeString(new Date())
58
+
private _perProjectBenchmarks = new Map<string, Map<string, TestBenchmarkTask>>()
57
59
58
60
constructor(options: BaseOptions = {}) {
59
61
this.isTTY = options.isTTY ?? isTTY
···
82
84
onTestRunStart(_specifications: ReadonlyArray<TestSpecification>): void {
83
85
this.start = performance.now()
84
86
this._timeStart = formatTimeString(new Date())
87
+
this._perProjectBenchmarks.clear()
85
88
}
86
89
87
90
onTestRunEnd(
···
97
100
this.ctx.logger.printNoTestFound(this.ctx.filenamePattern)
98
101
}
99
102
else {
103
+
this.printPerProjectBenchmarks()
100
104
this.reportSummary(files, errors)
101
105
}
102
106
}
···
104
108
onTestCaseResult(testCase: TestCase): void {
105
109
if (testCase.result().state === 'failed') {
106
110
this.logFailedTask(testCase.task)
111
+
}
112
+
}
113
+
114
+
onTestCaseBenchmark(testCase: TestCase, benchmark: TestBenchmark): void {
115
+
const projectName = testCase.project.name || ''
116
+
for (const task of benchmark.tasks) {
117
+
if (!task.perProject) {
118
+
continue
119
+
}
120
+
const benchKey = `${testCase.module.relativeModuleId} > ${testCase.fullName} > ${task.name}`
121
+
let projectMap = this._perProjectBenchmarks.get(benchKey)
122
+
if (!projectMap) {
123
+
projectMap = new Map()
124
+
this._perProjectBenchmarks.set(benchKey, projectMap)
125
+
}
126
+
projectMap.set(projectName, task)
107
127
}
108
128
}
109
129
···
206
226
const { duration = 0 } = test.diagnostic() || {}
207
227
const padding = this.getTestIndentation(test.task)
208
228
const suffix = this.getTestCaseSuffix(test)
229
+
const benchmarks = test.benchmarks()
230
+
// perProject tasks still appear in the inline table — they're additionally
231
+
// aggregated in the cross-project section at the end of the run
232
+
const inlineBenchmarks: TestBenchmark[] = benchmarks.filter(b => b.tasks.length > 0)
209
233
210
234
if (testResult.state === 'failed') {
211
235
this.log(c.red(` ${padding}${taskFail} ${this.getTestName(test.task, separator)}`) + suffix)
···
213
237
214
238
// also print slow tests
215
239
else if (duration > this.ctx.config.slowTestThreshold) {
216
-
this.log(` ${padding}${c.yellow(c.dim(F_CHECK))} ${this.getTestName(test.task, separator)} ${suffix}`)
240
+
this.log(` ${padding}${c.yellow(c.dim(F_CHECK))} ${this.getTestName(test.task, separator)}${suffix}`)
217
241
}
218
242
219
243
else if (this.ctx.config.hideSkippedTests && testResult.state === 'skipped' && test.options.mode !== 'todo') {
220
244
// Skipped tests are hidden when --hideSkippedTests
221
245
}
222
246
223
-
else if (this.renderSucceed || moduleState === 'failed') {
247
+
else if (this.renderSucceed || moduleState === 'failed' || inlineBenchmarks.length) {
224
248
this.log(` ${padding}${this.getStateSymbol(test)} ${this.getTestName(test.task, separator)}${suffix}`)
249
+
}
250
+
251
+
if (inlineBenchmarks.length > 0) {
252
+
this.printBenchmarkTable(inlineBenchmarks, padding)
225
253
}
226
254
}
227
255
···
545
573
546
574
const leakCount = this.printLeaksSummary()
547
575
548
-
if (this.ctx.config.mode === 'benchmark') {
549
-
this.reportBenchmarkSummary(files)
550
-
}
551
-
else {
552
-
this.reportTestSummary(files, errors, leakCount)
553
-
}
576
+
this.reportTestSummary(files, errors, leakCount)
554
577
}
555
578
556
579
reportTestSummary(files: File[], errors: unknown[], leakCount: number): void {
···
873
896
return leakWithStacks.size
874
897
}
875
898
876
-
reportBenchmarkSummary(files: File[]): void {
877
-
const benches = getTests(files)
878
-
const topBenches = benches.filter(i => i.result?.benchmark?.rank === 1)
899
+
protected printPerProjectBenchmarks(): void {
900
+
if (this._perProjectBenchmarks.size === 0) {
901
+
return
902
+
}
879
903
880
-
this.log(`\n${withLabel('cyan', 'BENCH', 'Summary\n')}`)
904
+
let hasComparable = false
905
+
for (const projectMap of this._perProjectBenchmarks.values()) {
906
+
if (projectMap.size > 1) {
907
+
hasComparable = true
908
+
break
909
+
}
910
+
}
911
+
if (!hasComparable) {
912
+
return
913
+
}
881
914
882
-
for (const bench of topBenches) {
883
-
const group = bench.suite || bench.file
915
+
this.log('')
916
+
this.log(divider(c.bold(c.bgBlue(` Cross-Project Benchmark Comparison `)), null, null, c.blue))
884
917
885
-
if (!group) {
918
+
for (const [benchName, projectMap] of this._perProjectBenchmarks) {
919
+
const tasks = [...projectMap.entries()]
920
+
.sort((a, b) => a[1].latency.mean - b[1].latency.mean)
921
+
.map(([projectName, task], index) => ({ ...task, name: projectName, rank: index + 1 }))
922
+
923
+
if (tasks.length <= 1) {
886
924
continue
887
925
}
888
926
889
-
const groupName = this.getFullName(group, separator)
890
-
const project = this.ctx.projects.find(p => p.name === bench.file.projectName)
927
+
this.log('')
928
+
this.log(` ${c.dim(benchName)}`)
929
+
this.printBenchmarkTable([{ name: benchName, tasks }], '', 'project')
930
+
}
891
931
892
-
this.log(` ${formatProjectName(project)}${bench.name}${c.dim(` - ${groupName}`)}`)
932
+
this.log('')
933
+
}
893
934
894
-
const siblings = group.tasks
895
-
.filter(i => i.meta.benchmark && i.result?.benchmark && i !== bench)
896
-
.sort((a, b) => a.result!.benchmark!.rank - b.result!.benchmark!.rank)
897
-
898
-
for (const sibling of siblings) {
899
-
const number = (sibling.result!.benchmark!.mean / bench.result!.benchmark!.mean).toFixed(2)
900
-
this.log(c.green(` ${number}x `) + c.gray('faster than ') + sibling.name)
935
+
protected printBenchmarkTable(benchmarks: readonly TestBenchmark[], basePadding: string, columnName = 'name'): void {
936
+
let printedCount = 0
937
+
for (const benchmark of benchmarks) {
938
+
const { tasks } = benchmark
939
+
if (tasks.length === 0) {
940
+
continue
901
941
}
902
942
903
-
this.log('')
943
+
if (printedCount > 0) {
944
+
this.log('')
945
+
}
946
+
947
+
const rows = tasks.map(t => renderBenchmarkRow(t))
948
+
const tableHead = [
949
+
columnName,
950
+
...BENCH_TABLE_HEAD,
951
+
]
952
+
const widths = computeBenchColumnWidths(tableHead, rows)
953
+
const indent = ` ${basePadding} `
954
+
955
+
this.log(`${indent}${padBenchRow(tableHead, widths).map(c.bold).join(' ')}`)
956
+
printedCount++
957
+
958
+
for (const task of tasks) {
959
+
const padded = padBenchRow(renderBenchmarkRow(task), widths)
960
+
let row = [
961
+
padded[0],
962
+
c.blue(padded[1]),
963
+
c.cyan(padded[2]),
964
+
c.cyan(padded[3]),
965
+
c.cyan(padded[4]),
966
+
c.cyan(padded[5]),
967
+
c.cyan(padded[6]),
968
+
c.cyan(padded[7]),
969
+
c.cyan(padded[8]),
970
+
c.dim(padded[9]),
971
+
c.dim(padded[10]),
972
+
].join(' ')
973
+
974
+
if (task.rank === 1 && tasks.length > 1) {
975
+
row += c.bold(c.green(' fastest'))
976
+
}
977
+
978
+
if (task.rank === tasks.length && tasks.length > 2) {
979
+
row += c.bold(c.gray(' slowest'))
980
+
}
981
+
982
+
this.log(`${indent}${row}`)
983
+
}
904
984
}
905
985
}
906
986
-6
packages/vitest/src/node/reporters/index.ts
-6
packages/vitest/src/node/reporters/index.ts
···
35
35
}
36
36
export type { BaseReporter, Reporter, TestRunEndReason }
37
37
38
-
export type { BenchmarkBuiltinReporters } from './benchmark'
39
-
export {
40
-
BenchmarkReporter,
41
-
BenchmarkReportsMap,
42
-
VerboseBenchmarkReporter,
43
-
} from './benchmark'
44
38
export type {
45
39
JsonAssertionResult,
46
40
JsonTestResult,
+3
-1
packages/vitest/src/node/reporters/json.ts
+3
-1
packages/vitest/src/node/reporters/json.ts
···
1
-
import type { Suite, TaskMeta, TaskState } from '@vitest/runner'
1
+
import type { Suite, TaskMeta, TaskState, TestBenchmark } from '@vitest/runner'
2
2
import type { SnapshotSummary } from '@vitest/snapshot'
3
3
import type { CoverageMap } from 'istanbul-lib-coverage'
4
4
import type { Vitest } from '../core'
···
40
40
failureMessages: Array<string> | null
41
41
location?: Callsite | null
42
42
tags: string[]
43
+
benchmarks: TestBenchmark[]
43
44
}
44
45
45
46
export interface JsonTestResult {
···
177
178
})()
178
179
: t.meta,
179
180
tags: t.tags || [],
181
+
benchmarks: t.benchmarks,
180
182
} satisfies JsonAssertionResult
181
183
})
182
184
+27
-1
packages/vitest/src/node/reporters/junit.ts
+27
-1
packages/vitest/src/node/reporters/junit.ts
···
11
11
import { getSuites } from '@vitest/runner/utils'
12
12
import { basename, dirname, relative, resolve } from 'pathe'
13
13
import { getOutputFile } from '../../utils/config-helpers'
14
+
import { renderBenchmarkTableText } from './renderers/benchmark-table'
14
15
import { IndentedLogger } from './renderers/indented-logger'
15
16
16
17
export interface ClassnameTemplateVariables {
···
319
320
})
320
321
}
321
322
323
+
async writeSystemOut(task: Task): Promise<void> {
324
+
const logs
325
+
= this.options.includeConsoleOutput && task.logs
326
+
? task.logs.filter(log => log.type === 'stdout')
327
+
: []
328
+
const benchmarks = task.type === 'test' ? task.benchmarks : []
329
+
330
+
if (logs.length === 0 && benchmarks.length === 0) {
331
+
return
332
+
}
333
+
334
+
await this.writeElement('system-out', {}, async () => {
335
+
for (const log of logs) {
336
+
await this.baseLog(escapeXML(log.content))
337
+
}
338
+
if (benchmarks.length > 0) {
339
+
if (logs.length > 0) {
340
+
await this.baseLog('')
341
+
}
342
+
await this.baseLog(escapeXML(renderBenchmarkTableText(benchmarks)))
343
+
}
344
+
})
345
+
}
346
+
322
347
private applyTemplate(
323
348
template: string | ((vars: ClassnameTemplateVariables) => string),
324
349
vars: ClassnameTemplateVariables,
···
368
393
time: getDuration(task),
369
394
},
370
395
async () => {
396
+
await this.writeSystemOut(task)
371
397
if (this.options.includeConsoleOutput) {
372
-
await this.writeLogs(task, 'out')
373
398
await this.writeLogs(task, 'err')
374
399
}
375
400
···
569
594
file: null as any,
570
595
annotations: [],
571
596
artifacts: [],
597
+
benchmarks: [],
572
598
} satisfies Task)
573
599
}
574
600
+10
packages/vitest/src/node/reporters/reported-tasks.ts
+10
packages/vitest/src/node/reporters/reported-tasks.ts
···
8
8
TaskMeta,
9
9
TestAnnotation,
10
10
TestArtifact,
11
+
TestBenchmark,
11
12
} from '@vitest/runner'
12
13
import type { SerializedError, TestError } from '@vitest/utils'
13
14
import type { DevEnvironment } from 'vite'
···
211
212
*/
212
213
public artifacts(): ReadonlyArray<TestArtifact> {
213
214
return [...this.task.artifacts]
215
+
}
216
+
217
+
/**
218
+
* @experimental
219
+
*
220
+
* A list of benchmarks performed during the test.
221
+
*/
222
+
public benchmarks(): ReadonlyArray<TestBenchmark> {
223
+
return [...this.task.benchmarks]
214
224
}
215
225
216
226
/**
+28
-26
packages/vitest/src/node/reporters/summary.ts
+28
-26
packages/vitest/src/node/reporters/summary.ts
···
2
2
import type { TestSpecification } from '../test-specification'
3
3
import type { Reporter } from '../types/reporter'
4
4
import type { Options as WindowRendererOptions } from './renderers/windowedRenderer'
5
-
import type { ReportedHookContext, TestCase, TestModule } from './reported-tasks'
5
+
import type { ReportedHookContext, TestCase, TestModule, TestSuite } from './reported-tasks'
6
6
import c from 'tinyrainbow'
7
7
import { F_POINTER, F_TREE_NODE_END, F_TREE_NODE_MIDDLE } from './renderers/figures'
8
8
import { formatProjectName, formatTime, formatTimeString, padSummaryTitle } from './renderers/utils'
···
36
36
visible: boolean
37
37
startTime: number
38
38
onFinish: () => void
39
-
hook?: Omit<SlowTask, 'hook'>
39
+
step?: Omit<SlowTask, 'step'>
40
40
}
41
41
42
42
interface RunningModule extends Pick<Counter, 'total' | 'completed'> {
43
43
filename: TestModule['task']['name']
44
44
projectName: TestModule['project']['name']
45
45
projectColor: TestModule['project']['color']
46
-
hook?: Omit<SlowTask, 'hook'>
46
+
step?: Omit<SlowTask, 'step'>
47
47
tests: Map<TestCase['id'], SlowTask>
48
48
meta: TestModule['task']['meta']
49
49
}
···
139
139
this.renderer.schedule()
140
140
}
141
141
142
-
onHookStart(options: ReportedHookContext): void {
143
-
const stats = this.getHookStats(options)
144
-
145
-
if (!stats) {
146
-
return
147
-
}
148
-
149
-
const hook = {
150
-
name: options.name,
142
+
private startStep(stats: RunningModule | SlowTask, name: string) {
143
+
const step = {
144
+
name,
151
145
visible: false,
152
146
startTime: performance.now(),
153
147
onFinish: () => {},
154
148
}
155
-
stats.hook?.onFinish?.()
156
-
stats.hook = hook
149
+
stats.step?.onFinish?.()
150
+
stats.step = step
157
151
158
152
if (!Number.isFinite(this.ctx.config.slowTestThreshold)) {
159
153
return
160
154
}
161
155
162
156
const timeout = setTimeout(() => {
163
-
hook.visible = true
157
+
step.visible = true
164
158
}, this.ctx.config.slowTestThreshold).unref()
165
159
166
-
hook.onFinish = () => clearTimeout(timeout)
160
+
step.onFinish = () => clearTimeout(timeout)
161
+
}
162
+
163
+
onHookStart(options: ReportedHookContext): void {
164
+
const stats = this.getStepStats(options.entity)
165
+
166
+
if (stats) {
167
+
this.startStep(stats, options.name)
168
+
}
167
169
}
168
170
169
171
onHookEnd(options: ReportedHookContext): void {
170
-
const stats = this.getHookStats(options)
172
+
const stats = this.getStepStats(options.entity)
171
173
172
-
if (stats?.hook?.name !== options.name) {
174
+
if (stats?.step?.name !== options.name) {
173
175
return
174
176
}
175
177
176
-
stats.hook.onFinish()
177
-
stats.hook.visible = false
178
+
stats.step.onFinish()
179
+
stats.step.visible = false
178
180
}
179
181
180
182
onTestCaseReady(test: TestCase): void {
···
203
205
: undefined
204
206
205
207
slowTest.onFinish = () => {
206
-
slowTest.hook?.onFinish()
208
+
slowTest.step?.onFinish()
207
209
clearTimeout(timeout)
208
210
}
209
211
···
283
285
this.renderer.schedule()
284
286
}
285
287
286
-
private getHookStats({ entity }: ReportedHookContext) {
288
+
private getStepStats(entity: TestSuite | TestModule | TestCase) {
287
289
// Track slow running hooks only on verbose mode
288
290
if (!this.options.verbose) {
289
291
return
···
317
319
)
318
320
319
321
const slowTasks = [
320
-
testFile.hook,
322
+
testFile.step,
321
323
...testFile.tests.values(),
322
324
].filter((t): t is SlowTask => t != null && t.visible)
323
325
···
331
333
+ c.bold(c.yellow(` ${formatTime(Math.max(0, elapsed))}`)),
332
334
)
333
335
334
-
if (task.hook?.visible) {
335
-
summary.push(c.bold(c.yellow(` ${F_TREE_NODE_END} `)) + task.hook.name)
336
+
if (task.step?.visible) {
337
+
summary.push(c.bold(c.yellow(` ${F_TREE_NODE_END} `)) + task.step.name)
336
338
}
337
339
}
338
340
}
···
367
369
}
368
370
369
371
const testFile = this.runningModules.get(id)
370
-
testFile?.hook?.onFinish()
372
+
testFile?.step?.onFinish()
371
373
testFile?.tests?.forEach(test => test.onFinish())
372
374
373
375
this.runningModules.delete(id)
+3
-31
packages/vitest/src/node/reporters/utils.ts
+3
-31
packages/vitest/src/node/reporters/utils.ts
···
3
3
import type { ResolvedConfig } from '../types/config'
4
4
import type { Reporter } from '../types/reporter'
5
5
import type { BlobReporter } from './blob'
6
-
import type { BenchmarkBuiltinReporters, BenchmarkReporter, BuiltinReporters, DefaultReporter, DotReporter, GithubActionsReporter, HangingProcessReporter, JsonReporter, JUnitReporter, TapReporter } from './index'
7
-
import { BenchmarkReportsMap, ReportersMap } from './index'
6
+
import type { BuiltinReporters, DefaultReporter, DotReporter, GithubActionsReporter, HangingProcessReporter, JsonReporter, JUnitReporter, TapReporter } from './index'
7
+
import { ReportersMap } from './index'
8
8
9
9
async function loadCustomReporterModule<C extends Reporter>(
10
10
path: string,
···
70
70
return Promise.all(promisedReporters)
71
71
}
72
72
73
-
function createBenchmarkReporters(
74
-
reporterReferences: Array<string | Reporter | BenchmarkBuiltinReporters>,
75
-
runner: ModuleRunner,
76
-
): Promise<(Reporter | BenchmarkReporter)[]> {
77
-
const promisedReporters = reporterReferences.map(
78
-
async (referenceOrInstance) => {
79
-
if (typeof referenceOrInstance === 'string') {
80
-
if (referenceOrInstance in BenchmarkReportsMap) {
81
-
const BuiltinReporter
82
-
= BenchmarkReportsMap[
83
-
referenceOrInstance as BenchmarkBuiltinReporters
84
-
]
85
-
return new BuiltinReporter()
86
-
}
87
-
else {
88
-
const CustomReporter = await loadCustomReporterModule(
89
-
referenceOrInstance,
90
-
runner,
91
-
)
92
-
return new CustomReporter()
93
-
}
94
-
}
95
-
return referenceOrInstance
96
-
},
97
-
)
98
-
return Promise.all(promisedReporters)
99
-
}
100
-
101
-
export { createBenchmarkReporters, createReporters }
73
+
export { createReporters }
+6
packages/vitest/src/node/reporters/verbose.ts
+6
packages/vitest/src/node/reporters/verbose.ts
···
44
44
this.printAnnotations(test, 'log', 3)
45
45
this.log()
46
46
}
47
+
48
+
const benchmarks = test.benchmarks()
49
+
const inlineBenchmarks = benchmarks.filter(b => b.tasks.length > 0)
50
+
if (inlineBenchmarks.length > 0) {
51
+
this.printBenchmarkTable(inlineBenchmarks, '')
52
+
}
47
53
}
48
54
}
+19
-33
packages/vitest/src/node/types/benchmark.ts
+19
-33
packages/vitest/src/node/types/benchmark.ts
···
1
-
import type { Arrayable } from '@vitest/utils'
2
-
3
-
import type { BenchmarkBuiltinReporters } from '../reporters'
4
-
import type { Reporter } from './reporter'
5
-
6
1
export interface BenchmarkUserOptions {
2
+
enabled?: boolean
3
+
7
4
/**
8
5
* Include globs for benchmark test files
9
6
*
···
13
10
14
11
/**
15
12
* Exclude globs for benchmark test files
16
-
* @default ['**\/node_modules/**', '**\/dist/**', '**\/cypress/**', '**\/.{idea,git,cache,output,temp}/**', '**\/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*']
13
+
* @default []
17
14
*/
18
15
exclude?: string[]
19
16
···
25
22
includeSource?: string[]
26
23
27
24
/**
28
-
* Custom reporters to use for output. Can contain one or more built-in reporter names, reporter instances,
29
-
* and/or paths to custom reporter files to import.
30
-
*
31
-
* @default ['default']
32
-
*/
33
-
reporters?: Arrayable<BenchmarkBuiltinReporters | Reporter | (string & {})>
34
-
35
-
/**
36
-
* @deprecated Use `benchmark.outputJson` instead
37
-
*/
38
-
outputFile?:
39
-
| string
40
-
| (Partial<Record<BenchmarkBuiltinReporters, string>>
41
-
& Record<string, string>)
42
-
43
-
/**
44
-
* benchmark output file to compare against
45
-
*/
46
-
compare?: string
47
-
48
-
/**
49
-
* benchmark output file
50
-
*/
51
-
outputJson?: string
52
-
53
-
/**
54
25
* Include `samples` array of benchmark results for API or custom reporter usages.
55
26
* This is disabled by default to reduce memory usage.
56
27
* @default false
57
28
*/
58
-
includeSamples?: boolean
29
+
retainSamples?: boolean
30
+
31
+
/**
32
+
* Disable warnings when a benchmark accesses module export getters too many times.
33
+
* @default false
34
+
*/
35
+
suppressExportGetterWarnings?: boolean
36
+
37
+
/**
38
+
* The name of the parent project that this benchmark project was cloned
39
+
* from. Populated automatically when Vitest creates the dedicated benchmark
40
+
* project for a parent project. Used by the runtime as the value for the
41
+
* `${projectName}` placeholder in `writeResult` / `bench.from()` paths.
42
+
* @internal
43
+
*/
44
+
projectName?: string
59
45
}
+5
-17
packages/vitest/src/node/types/config.ts
+5
-17
packages/vitest/src/node/types/config.ts
···
73
73
74
74
export type { HappyDOMOptions, JSDOMOptions }
75
75
76
-
export type VitestRunMode = 'test' | 'benchmark'
76
+
/**
77
+
* @deprecated
78
+
*/
79
+
export type VitestRunMode = 'test'
77
80
78
81
export interface ProjectName {
79
82
label: string
···
1094
1097
clearScreen?: boolean
1095
1098
1096
1099
/**
1097
-
* benchmark.compare option exposed at the top level for cli
1098
-
*/
1099
-
compare?: string
1100
-
1101
-
/**
1102
-
* benchmark.outputJson option exposed at the top level for cli
1103
-
*/
1104
-
outputJson?: string
1105
-
1106
-
/**
1107
1100
* Directory of blob reports to merge
1108
1101
* @default '.vitest/blob'
1109
1102
*/
···
1159
1152
| 'fileParallelism'
1160
1153
| 'tagsFilter'
1161
1154
> {
1162
-
mode: VitestRunMode
1163
-
1164
1155
name: ProjectName['label']
1165
1156
color?: ProjectName['color']
1166
1157
base?: string
···
1191
1182
cliExclude?: string[]
1192
1183
1193
1184
project: string[]
1194
-
benchmark?: Required<
1195
-
Omit<BenchmarkUserOptions, 'outputFile' | 'compare' | 'outputJson'>
1196
-
>
1197
-
& Pick<BenchmarkUserOptions, 'outputFile' | 'compare' | 'outputJson'>
1185
+
benchmark: Required<BenchmarkUserOptions>
1198
1186
shard?: {
1199
1187
index: number
1200
1188
count: number
+7
-1
packages/vitest/src/node/types/reporter.ts
+7
-1
packages/vitest/src/node/types/reporter.ts
···
1
-
import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact } from '@vitest/runner'
1
+
import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact, TestBenchmark } from '@vitest/runner'
2
2
import type { Awaitable, SerializedError } from '@vitest/utils'
3
3
import type { UserConsoleLog } from '../../types/general'
4
4
import type { Vitest } from '../core'
···
96
96
onHookEnd?: (hook: ReportedHookContext) => Awaitable<void>
97
97
98
98
onCoverage?: (coverage: unknown) => Awaitable<void>
99
+
100
+
/**
101
+
* @experimental
102
+
* Called after the benchmark is finished.
103
+
*/
104
+
onTestCaseBenchmark?: (testCase: TestCase, benchmark: TestBenchmark) => Awaitable<void>
99
105
}
+21
-7
packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts
+21
-7
packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts
···
5
5
ModuleRunnerContext,
6
6
ModuleRunnerImportMeta,
7
7
} from 'vite/module-runner'
8
+
import type { GetterTracker } from '../getter-tracker'
8
9
import type { VitestEvaluatedModules } from './evaluatedModules'
9
10
import type { ModuleExecutionInfo } from './moduleDebug'
10
11
import type { VitestVmOptions } from './moduleRunner'
···
30
31
getCurrentTestFilepath?: () => string | undefined
31
32
compiledFunctionArgumentsNames?: string[]
32
33
compiledFunctionArgumentsValues?: unknown[]
34
+
getterTracker?: GetterTracker
33
35
/**
34
36
* @internal
35
37
*/
···
43
45
44
46
private compiledFunctionArgumentsNames?: string[]
45
47
private compiledFunctionArgumentsValues: unknown[] = []
48
+
private getterTracker: GetterTracker | undefined
49
+
50
+
static EXPORTS_MAX_INVOCATIONS = 1_000_000
46
51
47
52
private primitives: {
48
53
Object: typeof Object
···
81
86
Reflect,
82
87
}
83
88
}
89
+
this.getterTracker = options.getterTracker
84
90
}
85
91
86
92
private convertIdToImportUrl(id: string) {
···
296
302
argumentsList.push(
297
303
// TODO@discuss deprecate in Vitest 5, remove in Vitest 6(?)
298
304
// backwards compat for vite-node
305
+
// https://github.com/vitest-dev/vitest/issues/10292
299
306
'__filename',
300
307
'__dirname',
301
308
'module',
···
333
340
? vm.runInContext(wrappedCode, this.vm.context, options)
334
341
: vm.runInThisContext(wrappedCode, options)
335
342
343
+
const __vite_ssr_exportName__ = context.__vite_ssr_exportName__
344
+
|| ((name: string, getter: () => unknown) => Object.defineProperty(context[ssrModuleExportsKey], name, {
345
+
enumerable: true,
346
+
configurable: true,
347
+
get: getter,
348
+
}))
349
+
350
+
let __vite_track_exportName__: ((name: string, getter: () => unknown) => void) | undefined
351
+
const getterTracker = this.getterTracker
352
+
if (getterTracker) {
353
+
__vite_track_exportName__ = getterTracker.createTracker(module.id, __vite_ssr_exportName__)
354
+
}
355
+
336
356
await initModule(
337
357
context[ssrModuleExportsKey],
338
358
context[ssrImportMetaKey],
339
359
context[ssrImportKey],
340
360
context[ssrDynamicImportKey],
341
361
context[ssrExportAllKey],
342
-
// vite 7 support, remove when vite 7+ is supported
343
-
context.__vite_ssr_exportName__
344
-
|| ((name: string, getter: () => unknown) => Object.defineProperty(context[ssrModuleExportsKey], name, {
345
-
enumerable: true,
346
-
configurable: true,
347
-
get: getter,
348
-
})),
362
+
__vite_track_exportName__ || __vite_ssr_exportName__,
349
363
350
364
cjsGlobals.__filename,
351
365
cjsGlobals.__dirname,
+1
packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts
+1
packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts
-180
packages/vitest/src/runtime/runners/benchmark.ts
-180
packages/vitest/src/runtime/runners/benchmark.ts
···
1
-
import type {
2
-
Suite,
3
-
Task,
4
-
TaskUpdateEvent,
5
-
VitestRunner,
6
-
VitestRunnerImportSource,
7
-
} from '@vitest/runner'
8
-
import type { ModuleRunner } from 'vite/module-runner'
9
-
import type { SerializedConfig } from '../config'
10
-
// import type { VitestExecutor } from '../execute'
11
-
import type {
12
-
Benchmark,
13
-
BenchmarkResult,
14
-
BenchTask,
15
-
} from '../types/benchmark'
16
-
import { updateTask as updateRunnerTask } from '@vitest/runner'
17
-
import { createDefer } from '@vitest/utils/helpers'
18
-
import { getSafeTimers } from '@vitest/utils/timers'
19
-
import { getBenchFn, getBenchOptions } from '../benchmark'
20
-
import { getWorkerState } from '../utils'
21
-
22
-
function createBenchmarkResult(name: string): BenchmarkResult {
23
-
return {
24
-
name,
25
-
rank: 0,
26
-
rme: 0,
27
-
samples: [] as number[],
28
-
} as BenchmarkResult
29
-
}
30
-
31
-
const benchmarkTasks = new WeakMap<Benchmark, import('tinybench').Task>()
32
-
33
-
async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) {
34
-
const { Task, Bench } = await runner.importTinybench()
35
-
36
-
const start = performance.now()
37
-
38
-
const benchmarkGroup: Benchmark[] = []
39
-
const benchmarkSuiteGroup = []
40
-
for (const task of suite.tasks) {
41
-
if (task.mode !== 'run' && task.mode !== 'queued') {
42
-
continue
43
-
}
44
-
45
-
if (task.meta?.benchmark) {
46
-
benchmarkGroup.push(task as Benchmark)
47
-
}
48
-
else if (task.type === 'suite') {
49
-
benchmarkSuiteGroup.push(task)
50
-
}
51
-
}
52
-
53
-
// run sub suites sequentially
54
-
for (const subSuite of benchmarkSuiteGroup) {
55
-
await runBenchmarkSuite(subSuite, runner)
56
-
}
57
-
58
-
if (benchmarkGroup.length) {
59
-
const defer = createDefer()
60
-
suite.result = {
61
-
state: 'run',
62
-
startTime: start,
63
-
benchmark: createBenchmarkResult(suite.name),
64
-
}
65
-
updateTask('suite-prepare', suite)
66
-
67
-
const addBenchTaskListener = (
68
-
task: InstanceType<typeof Task>,
69
-
benchmark: Benchmark,
70
-
) => {
71
-
task.addEventListener(
72
-
'complete',
73
-
(e) => {
74
-
const task = e.task
75
-
const taskRes = task.result!
76
-
const result = benchmark.result!.benchmark!
77
-
benchmark.result!.state = 'pass'
78
-
Object.assign(result, taskRes)
79
-
// compute extra stats and free raw samples as early as possible
80
-
const samples = result.samples
81
-
result.sampleCount = samples.length
82
-
result.median = samples.length % 2
83
-
? samples[Math.floor(samples.length / 2)]
84
-
: (samples[samples.length / 2] + samples[samples.length / 2 - 1]) / 2
85
-
if (!runner.config.benchmark?.includeSamples) {
86
-
result.samples.length = 0
87
-
}
88
-
updateTask('test-finished', benchmark)
89
-
},
90
-
{
91
-
once: true,
92
-
},
93
-
)
94
-
task.addEventListener(
95
-
'error',
96
-
(e) => {
97
-
const task = e.task
98
-
defer.reject(benchmark ? task.result!.error : e)
99
-
},
100
-
{
101
-
once: true,
102
-
},
103
-
)
104
-
}
105
-
106
-
benchmarkGroup.forEach((benchmark) => {
107
-
const options = getBenchOptions(benchmark)
108
-
const benchmarkInstance = new Bench(options)
109
-
110
-
const benchmarkFn = getBenchFn(benchmark)
111
-
112
-
benchmark.result = {
113
-
state: 'run',
114
-
startTime: start,
115
-
benchmark: createBenchmarkResult(benchmark.name),
116
-
}
117
-
118
-
const task = new Task(benchmarkInstance, benchmark.name, benchmarkFn)
119
-
benchmarkTasks.set(benchmark, task)
120
-
addBenchTaskListener(task, benchmark)
121
-
})
122
-
123
-
const { setTimeout } = getSafeTimers()
124
-
const tasks: [BenchTask, Benchmark][] = []
125
-
126
-
for (const benchmark of benchmarkGroup) {
127
-
const task = benchmarkTasks.get(benchmark)!
128
-
updateTask('test-prepare', benchmark)
129
-
await task.warmup()
130
-
tasks.push([
131
-
await new Promise<BenchTask>(resolve =>
132
-
setTimeout(async () => {
133
-
resolve(await task.run())
134
-
}),
135
-
),
136
-
benchmark,
137
-
])
138
-
}
139
-
140
-
suite.result!.duration = performance.now() - start
141
-
suite.result!.state = 'pass'
142
-
143
-
updateTask('suite-finished', suite)
144
-
defer.resolve(null)
145
-
146
-
await defer
147
-
}
148
-
149
-
function updateTask(event: TaskUpdateEvent, task: Task) {
150
-
updateRunnerTask(event, task, runner)
151
-
}
152
-
}
153
-
154
-
export class NodeBenchmarkRunner implements VitestRunner {
155
-
private moduleRunner!: ModuleRunner
156
-
157
-
constructor(public config: SerializedConfig) {}
158
-
159
-
async importTinybench(): Promise<typeof import('tinybench')> {
160
-
return await import('tinybench')
161
-
}
162
-
163
-
importFile(filepath: string, source: VitestRunnerImportSource): unknown {
164
-
if (source === 'setup') {
165
-
const moduleNode = getWorkerState().evaluatedModules.getModuleById(filepath)
166
-
if (moduleNode) {
167
-
getWorkerState().evaluatedModules.invalidateModule(moduleNode)
168
-
}
169
-
}
170
-
return this.moduleRunner.import(filepath)
171
-
}
172
-
173
-
async runSuite(suite: Suite): Promise<void> {
174
-
await runBenchmarkSuite(suite, this)
175
-
}
176
-
177
-
async runTask(): Promise<void> {
178
-
throw new Error('`test()` and `it()` is only available in test mode.')
179
-
}
180
-
}
+2
-5
packages/vitest/src/runtime/runners/index.ts
+2
-5
packages/vitest/src/runtime/runners/index.ts
···
6
6
import { rpc } from '../rpc'
7
7
import { loadDiffConfig, loadSnapshotSerializers } from '../setup-common'
8
8
import { getWorkerState } from '../utils'
9
-
import { NodeBenchmarkRunner } from './benchmark'
10
9
import { TestRunner } from './test'
11
10
12
11
async function getTestRunnerConstructor(
···
14
13
moduleRunner: TestModuleRunner,
15
14
): Promise<VitestRunnerConstructor> {
16
15
if (!config.runner) {
17
-
return (
18
-
config.mode === 'test' ? TestRunner : NodeBenchmarkRunner
19
-
) as any as VitestRunnerConstructor
16
+
return TestRunner as any as VitestRunnerConstructor
20
17
}
21
18
const mod = await moduleRunner.import(config.runner)
22
19
if (!mod.default && typeof mod.default !== 'function') {
···
60
57
loadDiffConfig(config, moduleRunner),
61
58
loadSnapshotSerializers(config, moduleRunner),
62
59
])
63
-
testRunner.config.diffOptions = diffOptions
60
+
testRunner.config._diffOptions = diffOptions
64
61
65
62
// patch some methods, so custom runners don't need to call RPC
66
63
const originalOnTaskUpdate = testRunner.onTaskUpdate
+30
-10
packages/vitest/src/runtime/runners/test.ts
+30
-10
packages/vitest/src/runtime/runners/test.ts
···
8
8
Task,
9
9
Test,
10
10
TestContext,
11
+
TestTryOptions,
11
12
VitestRunnerImportSource,
12
13
VitestRunner as VitestTestRunner,
13
14
} from '@vitest/runner'
15
+
import type { Bench as Tinybench, Task as TinybenchTask } from 'tinybench'
14
16
import type { ModuleRunner } from 'vite/module-runner'
15
17
import type { Traces } from '../../utils/traces'
18
+
import type { Bench } from '../benchmark'
16
19
import type { SerializedConfig } from '../config'
17
20
import { getState, GLOBAL_EXPECT, setState } from '@vitest/expect'
18
21
import {
···
29
32
import { inject } from '../../integrations/inject'
30
33
import { getSnapshotClient } from '../../integrations/snapshot/chai'
31
34
import { vi } from '../../integrations/vi'
32
-
import { getBenchFn, getBenchOptions } from '../benchmark'
35
+
import { createBench, kFinalize } from '../benchmark'
33
36
import { rpc } from '../rpc'
34
37
import { getWorkerState } from '../utils'
35
38
···
40
43
private cancelRun = false
41
44
42
45
private assertionsErrors = new WeakMap<Readonly<Task>, Error>()
46
+
private benchInstances = new WeakMap<Readonly<Task>, Bench>()
43
47
44
48
public pool: string = this.workerState.ctx.pool
45
-
private _otel!: Traces
49
+
/**
50
+
* @internal
51
+
*/
52
+
public _otel!: Traces
46
53
public viteEnvironment: string
47
54
private viteModuleRunner: boolean
48
55
···
83
90
this.workerState.onCleanup(listener)
84
91
}
85
92
86
-
onAfterRunFiles(): void {
93
+
onAfterRunFiles(_files: File[]): void {
87
94
this.snapshotClient.clear()
88
95
this.workerState.current = undefined
89
96
}
···
167
174
this.workerState.current = suite
168
175
}
169
176
170
-
onBeforeTryTask(test: Task): void {
177
+
onBeforeTryTask(test: Task, _options: TestTryOptions): void {
171
178
clearModuleMocks(this.config)
172
179
this.snapshotClient.clearTest(test.file.filepath, test.id)
173
180
setState(
···
185
192
}
186
193
187
194
onAfterTryTask(test: Test): void {
195
+
this.benchInstances.get(test)?.[kFinalize]()
188
196
const {
189
197
assertionCalls,
190
198
expectedAssertionsNumber,
···
229
237
Object.defineProperty(context, '_local', {
230
238
get() {
231
239
return _expect != null
240
+
},
241
+
})
242
+
let _bench: Bench | undefined
243
+
const runnerConfig = this.config
244
+
const benchInstances = this.benchInstances
245
+
Object.defineProperty(context, 'bench', {
246
+
get() {
247
+
if (!_bench) {
248
+
_bench = createBench(context.task, runnerConfig)
249
+
benchInstances.set(context.task, _bench)
250
+
}
251
+
return _bench
232
252
},
233
253
})
234
254
return context
···
281
301
static matchesTags: typeof matchesTags = matchesTags
282
302
283
303
/**
284
-
* @deprecated
304
+
* @experimental
305
+
* A function that runs tinybench tasks.
306
+
* Can be overriden to run tasks in a special environment.
285
307
*/
286
-
static getBenchFn: typeof getBenchFn = getBenchFn
287
-
/**
288
-
* @deprecated
289
-
*/
290
-
static getBenchOptions: typeof getBenchOptions = getBenchOptions
308
+
static async runBenchmarks(tinybench: Tinybench): Promise<TinybenchTask[]> {
309
+
return await tinybench.run()
310
+
}
291
311
}
292
312
293
313
function clearModuleMocks(config: SerializedConfig) {
-35
packages/vitest/src/runtime/types/benchmark.ts
-35
packages/vitest/src/runtime/types/benchmark.ts
···
1
-
import type { Test } from '@vitest/runner'
2
-
import type { ChainableFunction } from '@vitest/runner/utils'
3
-
import type {
4
-
Bench as BenchFactory,
5
-
Options as BenchOptions,
6
-
Task as BenchTask,
7
-
TaskResult as BenchTaskResult,
8
-
TaskResult as TinybenchResult,
9
-
} from 'tinybench'
10
-
11
-
export interface Benchmark extends Test {
12
-
meta: {
13
-
benchmark: true
14
-
result?: BenchTaskResult
15
-
}
16
-
}
17
-
18
-
export interface BenchmarkResult extends TinybenchResult {
19
-
name: string
20
-
rank: number
21
-
sampleCount: number
22
-
median: number
23
-
}
24
-
25
-
export type BenchFunction = (this: BenchFactory) => Promise<void> | void
26
-
type ChainableBenchmarkAPI = ChainableFunction<
27
-
'skip' | 'only' | 'todo',
28
-
(name: string | Function, fn?: BenchFunction, options?: BenchOptions) => void
29
-
>
30
-
export type BenchmarkAPI = ChainableBenchmarkAPI & {
31
-
skipIf: (condition: any) => ChainableBenchmarkAPI
32
-
runIf: (condition: any) => ChainableBenchmarkAPI
33
-
}
34
-
35
-
export { BenchFactory, BenchOptions, BenchTask, BenchTaskResult }
-67
test/e2e/fixtures/benchmarking/basic/base.bench.ts
-67
test/e2e/fixtures/benchmarking/basic/base.bench.ts
···
1
-
import { bench, describe } from 'vitest'
2
-
3
-
describe('sort', () => {
4
-
bench('normal', () => {
5
-
const x = [1, 5, 4, 2, 3]
6
-
x.sort((a, b) => {
7
-
return a - b
8
-
})
9
-
}, { iterations: 5, time: 0 })
10
-
11
-
bench('reverse', () => {
12
-
const x = [1, 5, 4, 2, 3]
13
-
x.reverse().sort((a, b) => {
14
-
return a - b
15
-
})
16
-
}, { iterations: 5, time: 0 })
17
-
18
-
// TODO: move to failed tests
19
-
// should not be collected
20
-
// it('test', () => {
21
-
// expect(1 + 1).toBe(3)
22
-
// })
23
-
})
24
-
25
-
function timeout(time: number) {
26
-
return new Promise((resolve) => {
27
-
setTimeout(resolve, time)
28
-
})
29
-
}
30
-
31
-
describe('timeout', () => {
32
-
bench('timeout100', async () => {
33
-
await timeout(100)
34
-
}, {
35
-
setup() {
36
-
37
-
},
38
-
teardown() {
39
-
40
-
},
41
-
...benchOptions
42
-
})
43
-
44
-
bench('timeout75', async () => {
45
-
await timeout(75)
46
-
}, benchOptions)
47
-
48
-
bench('timeout50', async () => {
49
-
await timeout(50)
50
-
}, benchOptions)
51
-
52
-
bench('timeout25', async () => {
53
-
await timeout(25)
54
-
}, benchOptions)
55
-
56
-
// TODO: move to failed tests
57
-
// test('reduce', () => {
58
-
// expect(1 - 1).toBe(2)
59
-
// })
60
-
})
61
-
62
-
const benchOptions = {
63
-
time: 0,
64
-
iterations: 3,
65
-
warmupIterations: 0,
66
-
warmupTime: 0,
67
-
}
-15
test/e2e/fixtures/benchmarking/basic/mode.bench.ts
-15
test/e2e/fixtures/benchmarking/basic/mode.bench.ts
···
1
-
import { bench, describe } from 'vitest'
2
-
3
-
describe.todo('unimplemented suite')
4
-
5
-
describe.skip('skipped', () => {
6
-
bench('skipped', () => {
7
-
throw new Error('should be skipped')
8
-
})
9
-
10
-
bench.todo('unimplemented test')
11
-
})
12
-
13
-
bench.skip('skipped', () => {
14
-
throw new Error('should be skipped')
15
-
})
-75
test/e2e/fixtures/benchmarking/basic/only.bench.ts
-75
test/e2e/fixtures/benchmarking/basic/only.bench.ts
···
1
-
import { bench, describe, expect, assert } from 'vitest'
2
-
3
-
const run = [false, false, false, false, false]
4
-
5
-
describe('a0', () => {
6
-
bench.only('0', () => {
7
-
run[0] = true
8
-
}, { iterations: 1, time: 0 })
9
-
bench('s0', () => {
10
-
expect(true).toBe(false)
11
-
})
12
-
})
13
-
14
-
describe('a1', () => {
15
-
describe('b1', () => {
16
-
describe('c1', () => {
17
-
bench.only('1', () => {
18
-
run[1] = true
19
-
}, { iterations: 1, time: 0 })
20
-
})
21
-
bench('s1', () => {
22
-
expect(true).toBe(false)
23
-
})
24
-
})
25
-
})
26
-
27
-
describe.only('a2', () => {
28
-
bench('2', () => {
29
-
run[2] = true
30
-
}, { iterations: 1, time: 0 })
31
-
})
32
-
33
-
bench('s2', () => {
34
-
expect(true).toBe(false)
35
-
})
36
-
37
-
describe.only('a3', () => {
38
-
describe('b3', () => {
39
-
bench('3', () => {
40
-
run[3] = true
41
-
}, { iterations: 1, time: 0 })
42
-
})
43
-
bench.skip('s3', () => {
44
-
expect(true).toBe(false)
45
-
})
46
-
})
47
-
48
-
describe('a4', () => {
49
-
describe.only('b4', () => {
50
-
bench('4', () => {
51
-
run[4] = true
52
-
}, { iterations: 1, time: 0 })
53
-
})
54
-
describe('sb4', () => {
55
-
bench('s4', () => {
56
-
expect(true).toBe(false)
57
-
})
58
-
})
59
-
})
60
-
61
-
bench.only(
62
-
'visited',
63
-
() => {
64
-
assert.deepEqual(run, [true, true, true, true, true])
65
-
},
66
-
{ iterations: 1, time: 0 },
67
-
)
68
-
69
-
bench.only(
70
-
'visited2',
71
-
() => {
72
-
assert.deepEqual(run, [true, true, true, true, true])
73
-
},
74
-
{ iterations: 1, time: 0 },
75
-
)
-7
test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts
-7
test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts
-3
test/e2e/fixtures/benchmarking/basic/vitest.config.ts
-3
test/e2e/fixtures/benchmarking/basic/vitest.config.ts
-13
test/e2e/fixtures/benchmarking/compare/basic.bench.ts
-13
test/e2e/fixtures/benchmarking/compare/basic.bench.ts
···
1
-
import { bench, describe } from 'vitest'
2
-
3
-
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
4
-
5
-
describe('suite', () => {
6
-
bench('sleep10', async () => {
7
-
await sleep(10)
8
-
}, { time: 20, iterations: 0 })
9
-
10
-
bench('sleep100', async () => {
11
-
await sleep(100);
12
-
}, { time: 200, iterations: 0 })
13
-
})
-3
test/e2e/fixtures/benchmarking/compare/vitest.config.ts
-3
test/e2e/fixtures/benchmarking/compare/vitest.config.ts
-12
test/e2e/fixtures/benchmarking/reporter/multiple.bench.ts
-12
test/e2e/fixtures/benchmarking/reporter/multiple.bench.ts
···
1
-
import { bench, describe } from 'vitest'
2
-
import { setTimeout } from 'node:timers/promises'
3
-
4
-
const options = { iterations: 1, warmupIterations: 1 }
5
-
6
-
bench('first', async () => {
7
-
await setTimeout(500)
8
-
}, options)
9
-
10
-
bench('second', async () => {
11
-
await setTimeout(500)
12
-
}, options)
-32
test/e2e/fixtures/benchmarking/reporter/summary.bench.ts
-32
test/e2e/fixtures/benchmarking/reporter/summary.bench.ts
···
1
-
import { bench, describe } from 'vitest'
2
-
3
-
describe('suite-a', () => {
4
-
bench('good', async () => {
5
-
await sleep(10)
6
-
}, options)
7
-
8
-
bench('bad', async () => {
9
-
await sleep(300)
10
-
}, options)
11
-
})
12
-
13
-
describe('suite-b', () => {
14
-
bench('good', async () => {
15
-
await sleep(25)
16
-
}, options)
17
-
18
-
describe('suite-b-nested', () => {
19
-
bench('good', async () => {
20
-
await sleep(50)
21
-
}, options)
22
-
})
23
-
})
24
-
25
-
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
26
-
27
-
const options = {
28
-
time: 0,
29
-
iterations: 2,
30
-
warmupIterations: 0,
31
-
warmupTime: 0,
32
-
}
-3
test/e2e/fixtures/benchmarking/reporter/vitest.config.ts
-3
test/e2e/fixtures/benchmarking/reporter/vitest.config.ts
-26
test/e2e/fixtures/benchmarking/sequential/f1.bench.ts
-26
test/e2e/fixtures/benchmarking/sequential/f1.bench.ts
···
1
-
import { bench, describe } from "vitest"
2
-
import { appendLog, benchOptions, sleepBench } from "./helper";
3
-
4
-
bench("B1", async () => {
5
-
await appendLog("F1 / B1")
6
-
await sleepBench();
7
-
}, benchOptions)
8
-
9
-
describe("S1", () => {
10
-
bench("B1", async () => {
11
-
await appendLog("F1 / S1 / B1")
12
-
await sleepBench();
13
-
}, benchOptions)
14
-
15
-
bench("B2", async () => {
16
-
await appendLog("F1 / S1 / B2")
17
-
await sleepBench();
18
-
}, benchOptions)
19
-
})
20
-
21
-
describe("S2", () => {
22
-
bench("B1", async () => {
23
-
await appendLog("F1 / S2 / B1")
24
-
await sleepBench();
25
-
}, benchOptions)
26
-
})
-9
test/e2e/fixtures/benchmarking/sequential/f2.bench.ts
-9
test/e2e/fixtures/benchmarking/sequential/f2.bench.ts
-19
test/e2e/fixtures/benchmarking/sequential/helper.ts
-19
test/e2e/fixtures/benchmarking/sequential/helper.ts
···
1
-
import fs from "node:fs";
2
-
3
-
const SLEEP_BENCH_MS = Number(process.env["SLEEP_BENCH_MS"] || 10);
4
-
const BENCH_ITERATIONS = Number(process.env["BENCH_ITERATIONS"] || 3);
5
-
6
-
export const sleepBench = () => new Promise(resolve => setTimeout(resolve, SLEEP_BENCH_MS))
7
-
8
-
export const testLogFile = new URL("./test.log", import.meta.url);
9
-
10
-
export async function appendLog(data: string) {
11
-
await fs.promises.appendFile(testLogFile, data + "\n");
12
-
}
13
-
14
-
export const benchOptions = {
15
-
time: 0,
16
-
iterations: BENCH_ITERATIONS,
17
-
warmupIterations: 0,
18
-
warmupTime: 0,
19
-
}
-6
test/e2e/fixtures/benchmarking/sequential/setup.ts
-6
test/e2e/fixtures/benchmarking/sequential/setup.ts
-10
test/e2e/fixtures/benchmarking/sequential/vitest.config.ts
-10
test/e2e/fixtures/benchmarking/sequential/vitest.config.ts
···
1
-
import { defineConfig } from "vitest/config"
2
-
3
-
// to see the difference better, increase sleep time and iterations e.g. by
4
-
// SLEEP_BENCH_MS=100 pnpm -C test/benchmark test bench -- --root fixtures/sequential --fileParallelism
5
-
6
-
export default defineConfig({
7
-
test: {
8
-
globalSetup: ["./setup.ts"]
9
-
}
10
-
});
+1
test/e2e/fixtures/custom-pool/pool/custom-pool.ts
+1
test/e2e/fixtures/custom-pool/pool/custom-pool.ts
+1
test/e2e/test/reporters/__snapshots__/json.test.ts.snap
+1
test/e2e/test/reporters/__snapshots__/json.test.ts.snap
+54
test/e2e/test/reporters/__snapshots__/reporters.test.ts.snap
+54
test/e2e/test/reporters/__snapshots__/reporters.test.ts.snap
···
130
130
"ancestorTitles": [
131
131
"suite",
132
132
],
133
+
"benchmarks": [],
133
134
"duration": 1.4422860145568848,
134
135
"failureMessages": [
135
136
"AssertionError: expected 2.23606797749979 to equal 2
···
158
159
"ancestorTitles": [
159
160
"suite",
160
161
],
162
+
"benchmarks": [],
161
163
"duration": 1.0237109661102295,
162
164
"failureMessages": [],
163
165
"fullName": "suite JSON",
···
170
172
"ancestorTitles": [
171
173
"suite",
172
174
],
175
+
"benchmarks": [],
173
176
"failureMessages": [],
174
177
"fullName": "suite async with timeout",
175
178
"meta": {},
···
181
184
"ancestorTitles": [
182
185
"suite",
183
186
],
187
+
"benchmarks": [],
184
188
"duration": 100.50598406791687,
185
189
"failureMessages": [],
186
190
"fullName": "suite timeout",
···
193
197
"ancestorTitles": [
194
198
"suite",
195
199
],
200
+
"benchmarks": [],
196
201
"duration": 20.184875011444092,
197
202
"failureMessages": [],
198
203
"fullName": "suite callback setup success ",
···
205
210
"ancestorTitles": [
206
211
"suite",
207
212
],
213
+
"benchmarks": [],
208
214
"duration": 0.33245420455932617,
209
215
"failureMessages": [],
210
216
"fullName": "suite callback test success ",
···
217
223
"ancestorTitles": [
218
224
"suite",
219
225
],
226
+
"benchmarks": [],
220
227
"duration": 19.738605976104736,
221
228
"failureMessages": [],
222
229
"fullName": "suite callback setup success done(false)",
···
229
236
"ancestorTitles": [
230
237
"suite",
231
238
],
239
+
"benchmarks": [],
232
240
"duration": 0.1923508644104004,
233
241
"failureMessages": [],
234
242
"fullName": "suite callback test success done(false)",
···
241
249
"ancestorTitles": [
242
250
"suite",
243
251
],
252
+
"benchmarks": [],
244
253
"failureMessages": [],
245
254
"fullName": "suite todo test",
246
255
"meta": {},
···
283
292
"ancestorTitles": [
284
293
"suite",
285
294
],
295
+
"benchmarks": [],
286
296
"duration": 1.4422860145568848,
287
297
"failureMessages": [
288
298
"AssertionError: expected 2.23606797749979 to equal 2
···
311
321
"ancestorTitles": [
312
322
"suite",
313
323
],
324
+
"benchmarks": [],
314
325
"duration": 1.0237109661102295,
315
326
"failureMessages": [],
316
327
"fullName": "suite JSON",
···
323
334
"ancestorTitles": [
324
335
"suite",
325
336
],
337
+
"benchmarks": [],
326
338
"failureMessages": [],
327
339
"fullName": "suite async with timeout",
328
340
"meta": {},
···
334
346
"ancestorTitles": [
335
347
"suite",
336
348
],
349
+
"benchmarks": [],
337
350
"duration": 100.50598406791687,
338
351
"failureMessages": [],
339
352
"fullName": "suite timeout",
···
346
359
"ancestorTitles": [
347
360
"suite",
348
361
],
362
+
"benchmarks": [],
349
363
"duration": 20.184875011444092,
350
364
"failureMessages": [],
351
365
"fullName": "suite callback setup success ",
···
358
372
"ancestorTitles": [
359
373
"suite",
360
374
],
375
+
"benchmarks": [],
361
376
"duration": 0.33245420455932617,
362
377
"failureMessages": [],
363
378
"fullName": "suite callback test success ",
···
370
385
"ancestorTitles": [
371
386
"suite",
372
387
],
388
+
"benchmarks": [],
373
389
"duration": 19.738605976104736,
374
390
"failureMessages": [],
375
391
"fullName": "suite callback setup success done(false)",
···
382
398
"ancestorTitles": [
383
399
"suite",
384
400
],
401
+
"benchmarks": [],
385
402
"duration": 0.1923508644104004,
386
403
"failureMessages": [],
387
404
"fullName": "suite callback test success done(false)",
···
394
411
"ancestorTitles": [
395
412
"suite",
396
413
],
414
+
"benchmarks": [],
397
415
"failureMessages": [],
398
416
"fullName": "suite todo test",
399
417
"meta": {},
···
441
459
"ancestorTitles": [
442
460
"suite",
443
461
],
462
+
"benchmarks": [],
444
463
"duration": 1.4422860145568848,
445
464
"failureMessages": [
446
465
"AssertionError: expected 2.23606797749979 to equal 2
···
469
488
"ancestorTitles": [
470
489
"suite",
471
490
],
491
+
"benchmarks": [],
472
492
"duration": 1.0237109661102295,
473
493
"failureMessages": [],
474
494
"fullName": "suite JSON",
···
481
501
"ancestorTitles": [
482
502
"suite",
483
503
],
504
+
"benchmarks": [],
484
505
"failureMessages": [],
485
506
"fullName": "suite async with timeout",
486
507
"meta": {},
···
492
513
"ancestorTitles": [
493
514
"suite",
494
515
],
516
+
"benchmarks": [],
495
517
"duration": 100.50598406791687,
496
518
"failureMessages": [],
497
519
"fullName": "suite timeout",
···
504
526
"ancestorTitles": [
505
527
"suite",
506
528
],
529
+
"benchmarks": [],
507
530
"duration": 20.184875011444092,
508
531
"failureMessages": [],
509
532
"fullName": "suite callback setup success ",
···
516
539
"ancestorTitles": [
517
540
"suite",
518
541
],
542
+
"benchmarks": [],
519
543
"duration": 0.33245420455932617,
520
544
"failureMessages": [],
521
545
"fullName": "suite callback test success ",
···
528
552
"ancestorTitles": [
529
553
"suite",
530
554
],
555
+
"benchmarks": [],
531
556
"duration": 19.738605976104736,
532
557
"failureMessages": [],
533
558
"fullName": "suite callback setup success done(false)",
···
540
565
"ancestorTitles": [
541
566
"suite",
542
567
],
568
+
"benchmarks": [],
543
569
"duration": 0.1923508644104004,
544
570
"failureMessages": [],
545
571
"fullName": "suite callback test success done(false)",
···
552
578
"ancestorTitles": [
553
579
"suite",
554
580
],
581
+
"benchmarks": [],
555
582
"failureMessages": [],
556
583
"fullName": "suite todo test",
557
584
"meta": {},
···
599
626
"ancestorTitles": [
600
627
"suite",
601
628
],
629
+
"benchmarks": [],
602
630
"duration": 1.4422860145568848,
603
631
"failureMessages": [
604
632
"AssertionError: expected 2.23606797749979 to equal 2
···
627
655
"ancestorTitles": [
628
656
"suite",
629
657
],
658
+
"benchmarks": [],
630
659
"duration": 1.0237109661102295,
631
660
"failureMessages": [],
632
661
"fullName": "suite JSON",
···
639
668
"ancestorTitles": [
640
669
"suite",
641
670
],
671
+
"benchmarks": [],
642
672
"failureMessages": [],
643
673
"fullName": "suite async with timeout",
644
674
"meta": {},
···
650
680
"ancestorTitles": [
651
681
"suite",
652
682
],
683
+
"benchmarks": [],
653
684
"duration": 100.50598406791687,
654
685
"failureMessages": [],
655
686
"fullName": "suite timeout",
···
662
693
"ancestorTitles": [
663
694
"suite",
664
695
],
696
+
"benchmarks": [],
665
697
"duration": 20.184875011444092,
666
698
"failureMessages": [],
667
699
"fullName": "suite callback setup success ",
···
674
706
"ancestorTitles": [
675
707
"suite",
676
708
],
709
+
"benchmarks": [],
677
710
"duration": 0.33245420455932617,
678
711
"failureMessages": [],
679
712
"fullName": "suite callback test success ",
···
686
719
"ancestorTitles": [
687
720
"suite",
688
721
],
722
+
"benchmarks": [],
689
723
"duration": 19.738605976104736,
690
724
"failureMessages": [],
691
725
"fullName": "suite callback setup success done(false)",
···
698
732
"ancestorTitles": [
699
733
"suite",
700
734
],
735
+
"benchmarks": [],
701
736
"duration": 0.1923508644104004,
702
737
"failureMessages": [],
703
738
"fullName": "suite callback test success done(false)",
···
710
745
"ancestorTitles": [
711
746
"suite",
712
747
],
748
+
"benchmarks": [],
713
749
"failureMessages": [],
714
750
"fullName": "suite todo test",
715
751
"meta": {},
···
757
793
"ancestorTitles": [
758
794
"suite",
759
795
],
796
+
"benchmarks": [],
760
797
"duration": 1.4422860145568848,
761
798
"failureMessages": [
762
799
"AssertionError: expected 2.23606797749979 to equal 2
···
785
822
"ancestorTitles": [
786
823
"suite",
787
824
],
825
+
"benchmarks": [],
788
826
"duration": 1.0237109661102295,
789
827
"failureMessages": [],
790
828
"fullName": "suite JSON",
···
797
835
"ancestorTitles": [
798
836
"suite",
799
837
],
838
+
"benchmarks": [],
800
839
"failureMessages": [],
801
840
"fullName": "suite async with timeout",
802
841
"meta": {},
···
808
847
"ancestorTitles": [
809
848
"suite",
810
849
],
850
+
"benchmarks": [],
811
851
"duration": 100.50598406791687,
812
852
"failureMessages": [],
813
853
"fullName": "suite timeout",
···
820
860
"ancestorTitles": [
821
861
"suite",
822
862
],
863
+
"benchmarks": [],
823
864
"duration": 20.184875011444092,
824
865
"failureMessages": [],
825
866
"fullName": "suite callback setup success ",
···
832
873
"ancestorTitles": [
833
874
"suite",
834
875
],
876
+
"benchmarks": [],
835
877
"duration": 0.33245420455932617,
836
878
"failureMessages": [],
837
879
"fullName": "suite callback test success ",
···
844
886
"ancestorTitles": [
845
887
"suite",
846
888
],
889
+
"benchmarks": [],
847
890
"duration": 19.738605976104736,
848
891
"failureMessages": [],
849
892
"fullName": "suite callback setup success done(false)",
···
856
899
"ancestorTitles": [
857
900
"suite",
858
901
],
902
+
"benchmarks": [],
859
903
"duration": 0.1923508644104004,
860
904
"failureMessages": [],
861
905
"fullName": "suite callback test success done(false)",
···
868
912
"ancestorTitles": [
869
913
"suite",
870
914
],
915
+
"benchmarks": [],
871
916
"failureMessages": [],
872
917
"fullName": "suite todo test",
873
918
"meta": {},
···
915
960
"ancestorTitles": [
916
961
"suite",
917
962
],
963
+
"benchmarks": [],
918
964
"duration": 1.4422860145568848,
919
965
"failureMessages": [
920
966
"AssertionError: expected 2.23606797749979 to equal 2
···
943
989
"ancestorTitles": [
944
990
"suite",
945
991
],
992
+
"benchmarks": [],
946
993
"duration": 1.0237109661102295,
947
994
"failureMessages": [],
948
995
"fullName": "suite JSON",
···
955
1002
"ancestorTitles": [
956
1003
"suite",
957
1004
],
1005
+
"benchmarks": [],
958
1006
"failureMessages": [],
959
1007
"fullName": "suite async with timeout",
960
1008
"meta": {},
···
966
1014
"ancestorTitles": [
967
1015
"suite",
968
1016
],
1017
+
"benchmarks": [],
969
1018
"duration": 100.50598406791687,
970
1019
"failureMessages": [],
971
1020
"fullName": "suite timeout",
···
978
1027
"ancestorTitles": [
979
1028
"suite",
980
1029
],
1030
+
"benchmarks": [],
981
1031
"duration": 20.184875011444092,
982
1032
"failureMessages": [],
983
1033
"fullName": "suite callback setup success ",
···
990
1040
"ancestorTitles": [
991
1041
"suite",
992
1042
],
1043
+
"benchmarks": [],
993
1044
"duration": 0.33245420455932617,
994
1045
"failureMessages": [],
995
1046
"fullName": "suite callback test success ",
···
1002
1053
"ancestorTitles": [
1003
1054
"suite",
1004
1055
],
1056
+
"benchmarks": [],
1005
1057
"duration": 19.738605976104736,
1006
1058
"failureMessages": [],
1007
1059
"fullName": "suite callback setup success done(false)",
···
1014
1066
"ancestorTitles": [
1015
1067
"suite",
1016
1068
],
1069
+
"benchmarks": [],
1017
1070
"duration": 0.1923508644104004,
1018
1071
"failureMessages": [],
1019
1072
"fullName": "suite callback test success done(false)",
···
1026
1079
"ancestorTitles": [
1027
1080
"suite",
1028
1081
],
1082
+
"benchmarks": [],
1029
1083
"failureMessages": [],
1030
1084
"fullName": "suite todo test",
1031
1085
"meta": {},
-17
packages/vitest/src/node/reporters/benchmark/index.ts
-17
packages/vitest/src/node/reporters/benchmark/index.ts
···
1
-
import { BenchmarkReporter } from './reporter'
2
-
import { VerboseBenchmarkReporter } from './verbose'
3
-
4
-
export {
5
-
BenchmarkReporter,
6
-
VerboseBenchmarkReporter,
7
-
}
8
-
9
-
export const BenchmarkReportsMap: {
10
-
default: typeof BenchmarkReporter
11
-
verbose: typeof VerboseBenchmarkReporter
12
-
} = {
13
-
default: BenchmarkReporter,
14
-
verbose: VerboseBenchmarkReporter,
15
-
}
16
-
17
-
export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap
-69
packages/vitest/src/node/reporters/benchmark/json-formatter.ts
-69
packages/vitest/src/node/reporters/benchmark/json-formatter.ts
···
1
-
import type { File } from '@vitest/runner'
2
-
import type { BenchmarkResult } from '../../../runtime/types/benchmark'
3
-
import { getFullName, getTasks } from '@vitest/runner/utils'
4
-
5
-
interface Report {
6
-
files: {
7
-
filepath: string
8
-
groups: Group[]
9
-
}[]
10
-
}
11
-
12
-
interface Group {
13
-
fullName: string
14
-
benchmarks: FormattedBenchmarkResult[]
15
-
}
16
-
17
-
export type FormattedBenchmarkResult = BenchmarkResult & {
18
-
id: string
19
-
}
20
-
21
-
export function createBenchmarkJsonReport(files: File[]): Report {
22
-
const report: Report = { files: [] }
23
-
24
-
for (const file of files) {
25
-
const groups: Group[] = []
26
-
27
-
for (const task of getTasks(file)) {
28
-
if (task?.type === 'suite') {
29
-
const benchmarks: FormattedBenchmarkResult[] = []
30
-
31
-
for (const t of task.tasks) {
32
-
const benchmark = t.meta.benchmark && t.result?.benchmark
33
-
34
-
if (benchmark) {
35
-
benchmarks.push({ id: t.id, ...benchmark, samples: [] })
36
-
}
37
-
}
38
-
39
-
if (benchmarks.length) {
40
-
groups.push({
41
-
fullName: getFullName(task, ' > '),
42
-
benchmarks,
43
-
})
44
-
}
45
-
}
46
-
}
47
-
48
-
report.files.push({
49
-
filepath: file.filepath,
50
-
groups,
51
-
})
52
-
}
53
-
54
-
return report
55
-
}
56
-
57
-
export function flattenFormattedBenchmarkReport(report: Report): Record<string, FormattedBenchmarkResult> {
58
-
const flat: Record<FormattedBenchmarkResult['id'], FormattedBenchmarkResult> = {}
59
-
60
-
for (const file of report.files) {
61
-
for (const group of file.groups) {
62
-
for (const t of group.benchmarks) {
63
-
flat[t.id] = t
64
-
}
65
-
}
66
-
}
67
-
68
-
return flat
69
-
}
-114
packages/vitest/src/node/reporters/benchmark/reporter.ts
-114
packages/vitest/src/node/reporters/benchmark/reporter.ts
···
1
-
import type { TaskResultPack } from '@vitest/runner'
2
-
import type { SerializedError } from '@vitest/utils'
3
-
import type { Vitest } from '../../core'
4
-
import type { TestRunEndReason } from '../../types/reporter'
5
-
import type { TestModule, TestSuite } from '../reported-tasks'
6
-
import fs from 'node:fs'
7
-
import { getFullName } from '@vitest/runner/utils'
8
-
import * as pathe from 'pathe'
9
-
import c from 'tinyrainbow'
10
-
import { DefaultReporter } from '../default'
11
-
import { formatProjectName, getStateSymbol, separator } from '../renderers/utils'
12
-
import { createBenchmarkJsonReport, flattenFormattedBenchmarkReport } from './json-formatter'
13
-
import { renderTable } from './tableRender'
14
-
15
-
export class BenchmarkReporter extends DefaultReporter {
16
-
compare?: Parameters<typeof renderTable>[0]['compare']
17
-
18
-
async onInit(ctx: Vitest): Promise<void> {
19
-
super.onInit(ctx)
20
-
21
-
if (this.ctx.config.benchmark?.compare) {
22
-
const compareFile = pathe.resolve(
23
-
this.ctx.config.root,
24
-
this.ctx.config.benchmark?.compare,
25
-
)
26
-
try {
27
-
this.compare = flattenFormattedBenchmarkReport(
28
-
JSON.parse(await fs.promises.readFile(compareFile, 'utf-8')),
29
-
)
30
-
}
31
-
catch (e) {
32
-
this.error(`Failed to read '${compareFile}'`, e)
33
-
}
34
-
}
35
-
}
36
-
37
-
onTaskUpdate(packs: TaskResultPack[]): void {
38
-
for (const pack of packs) {
39
-
const task = this.ctx.state.idMap.get(pack[0])
40
-
41
-
if (task?.type === 'suite' && task.result?.state !== 'run') {
42
-
task.tasks.filter(task => task.result?.benchmark)
43
-
.sort((benchA, benchB) => benchA.result!.benchmark!.mean - benchB.result!.benchmark!.mean)
44
-
.forEach((bench, idx) => {
45
-
bench.result!.benchmark!.rank = Number(idx) + 1
46
-
})
47
-
}
48
-
}
49
-
}
50
-
51
-
onTestSuiteResult(testSuite: TestSuite): void {
52
-
super.onTestSuiteResult(testSuite)
53
-
this.printSuiteTable(testSuite)
54
-
}
55
-
56
-
protected printTestModule(testModule: TestModule): void {
57
-
this.printSuiteTable(testModule)
58
-
}
59
-
60
-
private printSuiteTable(testTask: TestModule | TestSuite): void {
61
-
const state = testTask.state()
62
-
if (state === 'pending' || state === 'queued') {
63
-
return
64
-
}
65
-
66
-
const benches = testTask.task.tasks.filter(t => t.meta.benchmark)
67
-
const duration = testTask.task.result?.duration || 0
68
-
69
-
if (benches.length > 0 && benches.every(t => t.result?.state !== 'run' && t.result?.state !== 'queued')) {
70
-
let title = `\n ${getStateSymbol(testTask.task)} ${formatProjectName(testTask.project)}${getFullName(testTask.task, separator)}`
71
-
72
-
if (duration != null && duration > this.ctx.config.slowTestThreshold) {
73
-
title += c.yellow(` ${Math.round(duration)}${c.dim('ms')}`)
74
-
}
75
-
76
-
this.log(title)
77
-
this.log(renderTable({
78
-
tasks: benches,
79
-
level: 1,
80
-
shallow: true,
81
-
columns: this.ctx.logger.getColumns(),
82
-
compare: this.compare,
83
-
showHeap: this.ctx.config.logHeapUsage,
84
-
slowTestThreshold: this.ctx.config.slowTestThreshold,
85
-
}))
86
-
}
87
-
}
88
-
89
-
async onTestRunEnd(
90
-
testModules: ReadonlyArray<TestModule>,
91
-
unhandledErrors: ReadonlyArray<SerializedError>,
92
-
reason: TestRunEndReason,
93
-
): Promise<void> {
94
-
super.onTestRunEnd(testModules, unhandledErrors, reason)
95
-
96
-
// write output for future comparison
97
-
let outputFile = this.ctx.config.benchmark?.outputJson
98
-
99
-
if (outputFile) {
100
-
outputFile = pathe.resolve(this.ctx.config.root, outputFile)
101
-
const outputDirectory = pathe.dirname(outputFile)
102
-
103
-
if (!fs.existsSync(outputDirectory)) {
104
-
await fs.promises.mkdir(outputDirectory, { recursive: true })
105
-
}
106
-
107
-
const files = testModules.map(t => t.task.file)
108
-
const output = createBenchmarkJsonReport(files)
109
-
110
-
await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2))
111
-
this.log(`Benchmark report written to ${outputFile}`)
112
-
}
113
-
}
114
-
}
-221
packages/vitest/src/node/reporters/benchmark/tableRender.ts
-221
packages/vitest/src/node/reporters/benchmark/tableRender.ts
···
1
-
import type { Task } from '@vitest/runner'
2
-
import type { BenchmarkResult } from '../../../runtime/types/benchmark'
3
-
import type { FormattedBenchmarkResult } from './json-formatter'
4
-
import { stripVTControlCharacters } from 'node:util'
5
-
import { getTests } from '@vitest/runner/utils'
6
-
import { notNullish } from '@vitest/utils/helpers'
7
-
import c from 'tinyrainbow'
8
-
import { F_RIGHT } from '../renderers/figures'
9
-
import { getStateSymbol, truncateString } from '../renderers/utils'
10
-
11
-
const outputMap = new WeakMap<Task, string>()
12
-
13
-
function formatNumber(number: number) {
14
-
const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.')
15
-
16
-
return res[0].replace(/(?=(?:\d{3})+$)\B/g, ',') + (res[1] ? `.${res[1]}` : '')
17
-
}
18
-
19
-
const tableHead = [
20
-
'name',
21
-
'hz',
22
-
'min',
23
-
'max',
24
-
'mean',
25
-
'p75',
26
-
'p99',
27
-
'p995',
28
-
'p999',
29
-
'rme',
30
-
'samples',
31
-
]
32
-
33
-
function renderBenchmarkItems(result: BenchmarkResult) {
34
-
return [
35
-
result.name,
36
-
formatNumber(result.hz || 0),
37
-
formatNumber(result.min || 0),
38
-
formatNumber(result.max || 0),
39
-
formatNumber(result.mean || 0),
40
-
formatNumber(result.p75 || 0),
41
-
formatNumber(result.p99 || 0),
42
-
formatNumber(result.p995 || 0),
43
-
formatNumber(result.p999 || 0),
44
-
`±${(result.rme || 0).toFixed(2)}%`,
45
-
(result.sampleCount || 0).toString(),
46
-
]
47
-
}
48
-
49
-
function computeColumnWidths(results: BenchmarkResult[]): number[] {
50
-
const rows = [tableHead, ...results.map(v => renderBenchmarkItems(v))]
51
-
52
-
return Array.from(tableHead, (_, i) =>
53
-
Math.max(...rows.map(row => stripVTControlCharacters(row[i]).length)))
54
-
}
55
-
56
-
function padRow(row: string[], widths: number[]) {
57
-
return row.map(
58
-
(v, i) => (i ? v.padStart(widths[i], ' ') : v.padEnd(widths[i], ' ')), // name
59
-
)
60
-
}
61
-
62
-
function renderTableHead(widths: number[]) {
63
-
return ' '.repeat(3) + padRow(tableHead, widths).map(c.bold).join(' ')
64
-
}
65
-
66
-
function renderBenchmark(result: BenchmarkResult, widths: number[]) {
67
-
const padded = padRow(renderBenchmarkItems(result), widths)
68
-
return [
69
-
padded[0], // name
70
-
c.blue(padded[1]), // hz
71
-
c.cyan(padded[2]), // min
72
-
c.cyan(padded[3]), // max
73
-
c.cyan(padded[4]), // mean
74
-
c.cyan(padded[5]), // p75
75
-
c.cyan(padded[6]), // p99
76
-
c.cyan(padded[7]), // p995
77
-
c.cyan(padded[8]), // p999
78
-
c.dim(padded[9]), // rem
79
-
c.dim(padded[10]), // sample
80
-
].join(' ')
81
-
}
82
-
83
-
export function renderTable(
84
-
options: {
85
-
tasks: Task[]
86
-
level: number
87
-
shallow?: boolean
88
-
showHeap: boolean
89
-
columns: number
90
-
slowTestThreshold: number
91
-
compare?: Record<Task['id'], FormattedBenchmarkResult>
92
-
},
93
-
): string {
94
-
const output: string[] = []
95
-
96
-
const benchMap: Record<string, { current: BenchmarkResult; baseline?: BenchmarkResult }> = {}
97
-
98
-
for (const task of options.tasks) {
99
-
if (task.meta.benchmark && task.result?.benchmark) {
100
-
benchMap[task.id] = {
101
-
current: task.result.benchmark,
102
-
baseline: options.compare?.[task.id],
103
-
}
104
-
}
105
-
}
106
-
107
-
const benchCount = Object.entries(benchMap).length
108
-
109
-
const columnWidths = computeColumnWidths(
110
-
Object.values(benchMap)
111
-
.flatMap(v => [v.current, v.baseline])
112
-
.filter(notNullish),
113
-
)
114
-
115
-
let idx = 0
116
-
const padding = ' '.repeat(options.level ? 1 : 0)
117
-
118
-
for (const task of options.tasks) {
119
-
const duration = task.result?.duration
120
-
const bench = benchMap[task.id]
121
-
122
-
let prefix = ''
123
-
124
-
if (idx === 0 && task.meta?.benchmark) {
125
-
prefix += `${renderTableHead(columnWidths)}\n${padding}`
126
-
}
127
-
128
-
prefix += ` ${getStateSymbol(task)} `
129
-
130
-
let suffix = ''
131
-
132
-
if (task.type === 'suite') {
133
-
suffix += c.dim(` (${getTests(task).length})`)
134
-
}
135
-
136
-
if (task.mode === 'skip' || task.mode === 'todo') {
137
-
suffix += c.dim(c.gray(' [skipped]'))
138
-
}
139
-
140
-
if (duration != null) {
141
-
const color = duration > options.slowTestThreshold ? c.yellow : c.green
142
-
143
-
suffix += color(` ${Math.round(duration)}${c.dim('ms')}`)
144
-
}
145
-
146
-
if (options.showHeap && task.result?.heap != null) {
147
-
suffix += c.magenta(` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`)
148
-
}
149
-
150
-
if (bench) {
151
-
let body = renderBenchmark(bench.current, columnWidths)
152
-
153
-
if (options.compare && bench.baseline) {
154
-
if (bench.current.hz) {
155
-
const diff = bench.current.hz / bench.baseline.hz
156
-
const diffFixed = diff.toFixed(2)
157
-
158
-
if (diffFixed === '1.0.0') {
159
-
body += c.gray(` [${diffFixed}x]`)
160
-
}
161
-
162
-
if (diff > 1) {
163
-
body += c.blue(` [${diffFixed}x] ⇑`)
164
-
}
165
-
else {
166
-
body += c.red(` [${diffFixed}x] ⇓`)
167
-
}
168
-
}
169
-
output.push(padding + prefix + body + suffix)
170
-
171
-
const bodyBaseline = renderBenchmark(bench.baseline, columnWidths)
172
-
output.push(`${padding} ${bodyBaseline} ${c.dim('(baseline)')}`)
173
-
}
174
-
175
-
else {
176
-
if (bench.current.rank === 1 && benchCount > 1) {
177
-
body += c.bold(c.green(' fastest'))
178
-
}
179
-
180
-
if (bench.current.rank === benchCount && benchCount > 2) {
181
-
body += c.bold(c.gray(' slowest'))
182
-
}
183
-
184
-
output.push(padding + prefix + body + suffix)
185
-
}
186
-
}
187
-
else {
188
-
output.push(padding + prefix + task.name + suffix)
189
-
}
190
-
191
-
if (task.result?.state !== 'pass' && outputMap.get(task) != null) {
192
-
let data: string | undefined = outputMap.get(task)
193
-
194
-
if (typeof data === 'string') {
195
-
data = stripVTControlCharacters(data.trim().split('\n').filter(Boolean).pop()!)
196
-
if (data === '') {
197
-
data = undefined
198
-
}
199
-
}
200
-
201
-
if (data != null) {
202
-
const out = ` ${' '.repeat(options.level)}${F_RIGHT} ${data}`
203
-
output.push(c.gray(truncateString(out, options.columns)))
204
-
}
205
-
}
206
-
207
-
if (!options.shallow && task.type === 'suite' && task.tasks.length > 0) {
208
-
if (task.result?.state) {
209
-
output.push(renderTable({
210
-
...options,
211
-
tasks: task.tasks,
212
-
level: options.level + 1,
213
-
shallow: false,
214
-
}))
215
-
}
216
-
}
217
-
idx++
218
-
}
219
-
220
-
return output.filter(Boolean).join('\n')
221
-
}
-5
packages/vitest/src/node/reporters/benchmark/verbose.ts
-5
packages/vitest/src/node/reporters/benchmark/verbose.ts
+80
packages/vitest/src/node/reporters/renderers/benchmark-table.ts
+80
packages/vitest/src/node/reporters/renderers/benchmark-table.ts
···
1
+
import type { TestBenchmark, TestBenchmarkTask } from '@vitest/runner'
2
+
import { stripVTControlCharacters } from 'node:util'
3
+
4
+
export const BENCH_TABLE_HEAD: string[] = [
5
+
'hz',
6
+
'min',
7
+
'max',
8
+
'mean',
9
+
'p75',
10
+
'p99',
11
+
'p995',
12
+
'p999',
13
+
'rme',
14
+
'samples',
15
+
]
16
+
17
+
function formatBenchNumber(number: number): string {
18
+
const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.')
19
+
return res[0].replace(/(?=(?:\d{3})+$)\B/g, ',') + (res[1] ? `.${res[1]}` : '')
20
+
}
21
+
22
+
// Plain-text rendering of the benchmark table (no ANSI colors, no indent).
23
+
// Used by the junit reporter to embed benchmark data in <system-out>.
24
+
export function renderBenchmarkTableText(
25
+
benchmarks: readonly TestBenchmark[],
26
+
columnName = 'name',
27
+
): string {
28
+
const lines: string[] = []
29
+
for (const benchmark of benchmarks) {
30
+
const { tasks } = benchmark
31
+
if (tasks.length === 0) {
32
+
continue
33
+
}
34
+
if (lines.length > 0) {
35
+
lines.push('')
36
+
}
37
+
const rows = tasks.map(renderBenchmarkRow)
38
+
const head = [columnName, ...BENCH_TABLE_HEAD]
39
+
const widths = computeBenchColumnWidths(head, rows)
40
+
lines.push(padBenchRow(head, widths).join(' '))
41
+
for (const task of tasks) {
42
+
let row = padBenchRow(renderBenchmarkRow(task), widths).join(' ')
43
+
if (task.rank === 1 && tasks.length > 1) {
44
+
row += ' fastest'
45
+
}
46
+
if (task.rank === tasks.length && tasks.length > 2) {
47
+
row += ' slowest'
48
+
}
49
+
lines.push(row)
50
+
}
51
+
}
52
+
return lines.join('\n')
53
+
}
54
+
55
+
export function renderBenchmarkRow(task: TestBenchmarkTask): string[] {
56
+
return [
57
+
task.name,
58
+
formatBenchNumber(task.throughput.mean || 0),
59
+
formatBenchNumber(task.latency.min || 0),
60
+
formatBenchNumber(task.latency.max || 0),
61
+
formatBenchNumber(task.latency.mean || 0),
62
+
formatBenchNumber(task.latency.p75 || 0),
63
+
formatBenchNumber(task.latency.p99 || 0),
64
+
formatBenchNumber(task.latency.p995 || 0),
65
+
formatBenchNumber(task.latency.p999 || 0),
66
+
`\u00B1${(task.latency.rme || 0).toFixed(2)}%`,
67
+
String(task.latency.samplesCount || 0),
68
+
]
69
+
}
70
+
71
+
export function computeBenchColumnWidths(header: string[], rows: string[][]): number[] {
72
+
const allRows = [header, ...rows]
73
+
return Array.from(header, (_, i) => Math.max(...allRows.map(row => stripVTControlCharacters(row[i]).length)))
74
+
}
75
+
76
+
export function padBenchRow(row: string[], widths: number[]): string[] {
77
+
return row.map(
78
+
(v, i) => (i === 0 ? v.padEnd(widths[i]) : v.padStart(widths[i])),
79
+
)
80
+
}