feat(benchmark): add pluggable benchmark provider API · vitest.dev/vitest@3dbe73a

+1

packages/vitest/src/defaults.ts

··· 20 20 exclude: defaultExclude, 21 21 includeSource: [], 22 22 retainSamples: false, 23 + provider: 'default', 23 24 suppressExportGetterWarnings: false, 24 25 // Populated automatically when Vitest clones the parent project; the default 25 26 // here applies to the (unused) raw config that's never run as a benchmark.

+6

packages/vitest/src/node/config/resolveConfig.ts

··· 294 294 ...benchmarkConfigDefaults, 295 295 ...resolved.benchmark, 296 296 } 297 + if (resolved.benchmark.provider && resolved.benchmark.provider !== 'default') { 298 + resolved.benchmark.provider = resolvePath( 299 + resolved.benchmark.provider, 300 + resolved.root, 301 + ) 302 + } 297 303 298 304 const inspector = resolved.inspect || resolved.inspectBrk 299 305

+1

packages/vitest/src/node/config/serializeConfig.ts

··· 134 134 benchmark: { 135 135 enabled: config.benchmark.enabled, 136 136 retainSamples: config.benchmark.retainSamples, 137 + provider: config.benchmark.provider, 137 138 suppressExportGetterWarnings: config.benchmark.suppressExportGetterWarnings, 138 139 projectName: config.benchmark.projectName, 139 140 },

+12

packages/vitest/src/node/types/benchmark.ts

··· 29 29 retainSamples?: boolean 30 30 31 31 /** 32 + * The benchmark provider that executes registered benchmarks and produces 33 + * their results. Either the built-in `'default'` provider (backed by 34 + * tinybench) or a path to a module whose default export implements 35 + * `BenchmarkProviderModule`. The path is resolved relative to the project 36 + * root. 37 + * 38 + * @default 'default' 39 + * @experimental 40 + */ 41 + provider?: 'default' | (string & {}) 42 + 43 + /** 32 44 * Disable warnings when a benchmark accesses module export getters too many times. 33 45 * @default false 34 46 */

+7

packages/vitest/src/public/index.ts

··· 38 38 export type { 39 39 Bench, 40 40 BenchCompareOptions, 41 + BenchFn, 41 42 BenchFnOptions, 42 43 BenchFromSource, 44 + BenchHooks, 45 + BenchmarkGroup, 46 + BenchmarkProvider, 47 + BenchmarkProviderModule, 43 48 BenchRegistration, 49 + BenchRegistrationInput, 44 50 BenchResult, 51 + BenchRunOptions, 45 52 BenchStorage, 46 53 } from '../runtime/benchmark' 47 54

+203 -102

packages/vitest/src/runtime/benchmark.ts

··· 5 5 TaskResultCompleted, 6 6 TaskResultRuntimeInfo, 7 7 TaskResultTimestampProviderInfo, 8 - Task as TinybenchTask, 9 8 } from 'tinybench' 10 9 import type { SerializedConfig } from './config' 11 10 import type { BaselineData, Test, TestBenchmark, TestBenchmarkTask } from './runner/types' 12 11 import { isAbsolute, relative } from 'pathe' 13 - import { Bench as Tinybench } from 'tinybench' 14 12 import c from 'tinyrainbow' 13 + import { createDefaultBenchmarkProvider } from './benchmark/default-provider' 15 14 import { rpc } from './rpc' 16 - import { TestRunner } from './runners/test' 17 15 import { getWorkerState } from './utils' 18 16 19 - const now = globalThis.performance 20 - ? globalThis.performance.now.bind(globalThis.performance) 21 - : Date.now 22 - 23 17 const kRegistration: unique symbol = Symbol('registration') 24 18 const kFromSource: unique symbol = Symbol('fromSource') 25 19 const kPerProject: unique symbol = Symbol('perProject') ··· 30 24 [K in keyof T]: T[K] extends BenchRegistration<infer N> ? N : never 31 25 }[number], never> 32 26 33 - // We throw an error if benchmark did not complete, so it will always be TaskResultCompleted 34 - export type BenchResult = TaskResultCompleted & TaskResultRuntimeInfo & TaskResultTimestampProviderInfo 27 + /** 28 + * A benchmarked function. Vitest-owned and engine-agnostic. 29 + */ 30 + export type BenchFn = Fn 31 + 32 + /** 33 + * Per-benchmark lifecycle hooks passed to `bench(name, hooks, fn)`. 34 + */ 35 + export type BenchHooks = FnOptions 36 + 37 + /** 38 + * Engine-neutral run options forwarded to a {@link BenchmarkProvider}. These 39 + * come from the optional trailing options argument of `bench.compare()` and 40 + * `registration.run()`. 41 + */ 42 + export type BenchRunOptions = BenchCompareOptions 43 + 44 + /** 45 + * The result of a single benchmark, produced by a {@link BenchmarkProvider}. 46 + * Structurally compatible with tinybench's task statistics so it flows 47 + * unchanged into the reporter, `bench.from()` baselines and comparison tables. 48 + * The `name` matches the corresponding {@link BenchRegistration}. 49 + */ 50 + export interface BenchResult extends TaskResultCompleted, TaskResultRuntimeInfo, TaskResultTimestampProviderInfo { 51 + /** The registered benchmark name this result belongs to. */ 52 + name: string 53 + } 54 + 55 + /** 56 + * A benchmark defined by `bench()` (or `bench.from()`), as seen by a 57 + * {@link BenchmarkProvider}. `fn`/`fnOpts` are absent for `bench.from()` 58 + * registrations, which are resolved from a stored baseline instead of run. 59 + */ 60 + export interface BenchRegistrationInput { 61 + name: string 62 + fn: BenchFn 63 + fnOpts?: BenchHooks 64 + } 65 + 66 + /** 67 + * The set of benchmarks registered by a single test, handed to 68 + * {@link BenchmarkProvider.run}. 69 + */ 70 + export interface BenchmarkGroup { 71 + /** The test that registered these benchmarks. */ 72 + test: Test 73 + /** The resolved benchmark configuration for the current project. */ 74 + config: SerializedConfig['benchmark'] 75 + /** The runnable registrations, in registration order. */ 76 + registrations: BenchRegistrationInput[] 77 + /** Engine-neutral run options, when provided by the caller. */ 78 + options?: BenchRunOptions 79 + } 80 + 81 + /** 82 + * Executes the benchmarks of a single test and returns their results. 83 + * 84 + * The returned array is the sole source of results: it's what `bench().run()` 85 + * resolves to and what the reporter serializes. Results are matched to 86 + * registrations by `name`; return one {@link BenchResult} per registration. 87 + * 88 + * @experimental 89 + */ 90 + export interface BenchmarkProvider { 91 + run: (group: BenchmarkGroup) => Promise<BenchResult[]> 92 + } 93 + 94 + /** 95 + * The module contract a custom benchmark provider must satisfy. The module's 96 + * default export must be an object of this shape. 97 + * 98 + * @experimental 99 + */ 100 + export interface BenchmarkProviderModule { 101 + /** Factory that creates the provider used to run benchmarks. */ 102 + getProvider: () => BenchmarkProvider | Promise<BenchmarkProvider> 103 + } 104 + 105 + interface BenchmarkProviderLoader { 106 + import: (id: string) => Promise<Record<string, any>> 107 + } 108 + 109 + let cachedProvider: Promise<BenchmarkProvider> | undefined 110 + 111 + async function loadProviderModule( 112 + provider: string, 113 + loader: BenchmarkProviderLoader, 114 + ): Promise<BenchmarkProviderModule> { 115 + let mod: Record<string, any> 116 + try { 117 + mod = await loader.import(provider) 118 + } 119 + catch (error) { 120 + throw new Error( 121 + `Failed to load benchmark provider from "${provider}".`, 122 + { cause: error }, 123 + ) 124 + } 125 + if (mod.default == null) { 126 + throw new Error( 127 + `Benchmark provider loaded from "${provider}" did not have a default export.`, 128 + ) 129 + } 130 + return mod.default 131 + } 132 + 133 + /** 134 + * Resolves the benchmark provider for the current worker, importing a custom 135 + * provider module on first use. The result is cached for the lifetime of the 136 + * worker so a custom provider is imported at most once. 137 + */ 138 + export function resolveBenchmarkProvider( 139 + config: SerializedConfig, 140 + loader: BenchmarkProviderLoader, 141 + ): Promise<BenchmarkProvider> { 142 + if (!cachedProvider) { 143 + const provider = config.benchmark.provider 144 + cachedProvider = provider === 'default' || !provider 145 + ? Promise.resolve(createDefaultBenchmarkProvider(config)) 146 + : Promise.resolve(loadProviderModule(provider, loader)).then(mod => mod.getProvider()) 147 + } 148 + return cachedProvider 149 + } 35 150 36 151 export interface BenchStorage<T extends string> { 37 152 get: (name: T) => BenchResult ··· 40 155 export type { BenchOptions as BenchCompareOptions } from 'tinybench' 41 156 42 157 /** 43 - * Options accepted by `bench(name, options, fn)`. Extends tinybench's 44 - * `FnOptions` with Vitest-specific fields. 158 + * Options accepted by `bench(name, options, fn)`. Extends the per-benchmark 159 + * lifecycle hooks with Vitest-specific fields. 45 160 */ 46 - export interface BenchFnOptions extends FnOptions { 161 + export interface BenchFnOptions extends BenchHooks { 47 162 /** 48 163 * Path (relative to the project root) where the benchmark result is written 49 164 * after a successful run. The string `${projectName}` is substituted with ··· 65 180 /** 66 181 * The benchmark function. Absent for registrations created via `bench.from()`. 67 182 */ 68 - fn?: Fn 183 + fn?: BenchFn 69 184 /** 70 185 * Per-benchmark options (`beforeEach`, `beforeAll`, etc.). Absent for 71 186 * registrations created via `bench.from()`. 72 187 */ 73 - fnOpts?: FnOptions 188 + fnOpts?: BenchHooks 74 189 /** 75 190 * @internal 76 191 */ 77 192 [kRegistration]: true 78 - run: (options?: BenchCompareOptions) => Promise<BenchResult> 193 + run: (options?: BenchRunOptions) => Promise<BenchResult> 79 194 } 80 195 81 196 interface BenchCompare { 82 197 <Args extends BenchRegistration<any>[]>(...args: Args): Promise<BenchStorage<ExtractBenchNames<Args>>> 83 - <Args extends BenchRegistration<any>[]>(...args: [...Args, BenchCompareOptions]): Promise<BenchStorage<ExtractBenchNames<Args>>> 198 + <Args extends BenchRegistration<any>[]>(...args: [...Args, BenchRunOptions]): Promise<BenchStorage<ExtractBenchNames<Args>>> 84 199 } 85 200 86 201 interface BenchFactory { 87 - <Name extends string>(name: Name | Function, fn: Fn): BenchRegistration<Name> 88 - <Name extends string>(name: Name | Function, options: BenchFnOptions, fn: Fn): BenchRegistration<Name> 202 + <Name extends string>(name: Name | Function, fn: BenchFn): BenchRegistration<Name> 203 + <Name extends string>(name: Name | Function, options: BenchFnOptions, fn: BenchFn): BenchRegistration<Name> 89 204 } 90 205 91 206 export interface BenchFromSource { ··· 104 219 } 105 220 106 221 interface RunnableRegistration<Name extends string> extends BenchRegistration<Name> { 107 - fn: Fn 108 - fnOpts?: FnOptions 222 + fn: BenchFn 223 + fnOpts?: BenchHooks 109 224 [kWriteResult]?: string 110 225 [kPerProject]?: true 111 226 } ··· 122 237 return template.replace(/\$\{projectName\}/g, projectName ?? '') 123 238 } 124 239 125 - export function createBench(test: Test, config: SerializedConfig): Bench { 126 - let benchIdx = 0 240 + export function createBench( 241 + test: Test, 242 + config: SerializedConfig, 243 + loader: BenchmarkProviderLoader, 244 + ): Bench { 127 245 const pending = new Set<BenchRegistration<any>>() 128 - const createTinybench = (options?: BenchCompareOptions) => { 129 - const currentIndex = ++benchIdx 130 - return new Tinybench({ 131 - signal: test.context.signal, 132 - name: `${test.fullTestName} ${currentIndex}`, 133 - retainSamples: config.benchmark.retainSamples, 134 - ...options, 135 - now, 136 - }) 137 - } 138 246 139 247 const resolveTemplate = (template: string) => substitutePath(template, config.benchmark.projectName) 140 248 ··· 164 272 }) 165 273 166 274 const createCompareStorage = <T extends string>( 167 - bench: Tinybench, 275 + results: Map<string, BenchResult>, 168 276 fromResults?: Map<string, BaselineData>, 169 277 ): BenchStorage<T> => { 170 278 return { ··· 173 281 if (stored) { 174 282 return stored as BenchResult 175 283 } 176 - const task = bench.getTask(name) 177 - if (!task) { 284 + const result = results.get(name) 285 + if (!result) { 178 286 throw new Error(`task "${name}" was not defined`) 179 287 } 180 - return task.result as BenchResult 288 + return result 181 289 }, 182 290 } 183 291 } ··· 185 293 interface TaskMeta { perProject?: true } 186 294 187 295 const serializeBenchmark = ( 188 - tinybenchTasks: TinybenchTask[], 189 - name: string | undefined, 296 + results: BenchResult[], 297 + name: string, 190 298 taskMeta?: Map<string, TaskMeta>, 191 299 fromTasks?: TestBenchmarkTask[], 192 300 ): TestBenchmark => { 193 - const tasks: TestBenchmarkTask[] = tinybenchTasks.map((t) => { 194 - const result = t.result 195 - if (result.state === 'errored') { 196 - throw result.error 197 - } 198 - if (result.state !== 'completed') { 199 - throw new Error(`task "${t.name}" did not complete: received "${result.state}"`) 200 - } 201 - return { 202 - name: t.name, 203 - latency: result.latency, 204 - throughput: result.throughput, 205 - period: result.period, 206 - totalTime: result.totalTime, 207 - rank: 0, 208 - ...taskMeta?.get(t.name), 209 - } 210 - }) 301 + const tasks: TestBenchmarkTask[] = results.map(result => ({ 302 + name: result.name, 303 + latency: result.latency, 304 + throughput: result.throughput, 305 + period: result.period, 306 + totalTime: result.totalTime, 307 + rank: 0, 308 + ...taskMeta?.get(result.name), 309 + })) 211 310 if (fromTasks) { 212 311 tasks.push(...fromTasks) 213 312 } ··· 216 315 task.rank = idx + 1 217 316 }) 218 317 return { 219 - name: name || test.fullTestName, 318 + name, 220 319 tasks, 221 320 } 222 321 } 223 322 224 323 const recordBenchmark = async ( 225 - tinybenchTasks: TinybenchTask[], 226 - name: string | undefined, 324 + results: BenchResult[], 325 + name: string, 227 326 taskMeta?: Map<string, TaskMeta>, 228 327 fromTasks?: TestBenchmarkTask[], 229 328 ) => { 230 - const serializedBenchmark = serializeBenchmark(tinybenchTasks, name, taskMeta, fromTasks) 329 + const serializedBenchmark = serializeBenchmark(results, name, taskMeta, fromTasks) 231 330 test.benchmarks.push(serializedBenchmark) 232 331 await rpc().onTestBenchmark(test.id, serializedBenchmark) 233 332 } ··· 243 342 await rpc().writeBenchmarkResult(resolved, data) 244 343 } 245 344 246 - const runBenchmarks = async (tinybench: Tinybench) => { 345 + const groupName = (options: BenchRunOptions | undefined) => options?.name ?? test.fullTestName 346 + 347 + const runGroup = async ( 348 + registrations: BenchRegistrationInput[], 349 + options: BenchRunOptions | undefined, 350 + ): Promise<Map<string, BenchResult>> => { 247 351 const workerState = getWorkerState() 248 352 const getterTracker = workerState.getterTracker 249 353 getterTracker?.resetInvocations() 250 354 try { 251 - return await TestRunner.runBenchmarks(tinybench) 355 + const provider = await resolveBenchmarkProvider(config, loader) 356 + const results = await provider.run({ test, config: config.benchmark, registrations, options }) 357 + const byName = new Map<string, BenchResult>() 358 + for (const result of results) { 359 + byName.set(result.name, result) 360 + } 361 + return byName 252 362 } 253 363 finally { 254 364 const excessiveInvocations = config.benchmark.suppressExportGetterWarnings ··· 261 371 console.warn( 262 372 [ 263 373 c.yellow(c.bold('Benchmark Warning')), 264 - `Benchmark ${c.bold(`"${tinybench.name}"`)} accessed module export getters too many times.`, 374 + `Benchmark ${c.bold(`"${groupName(options)}"`)} accessed module export getters too many times.`, 265 375 '', 266 376 'This can make results unreliable because export getters add overhead.', 267 377 'See https://vitest.dev/guide/benchmarking#module-runner-overhead', ··· 276 386 277 387 const runSingle = async ( 278 388 name: string, 279 - fn: Fn, 280 - fnOpts: FnOptions | undefined, 281 - options: BenchCompareOptions | undefined, 389 + fn: BenchFn, 390 + fnOpts: BenchHooks | undefined, 391 + options: BenchRunOptions | undefined, 282 392 meta: TaskMeta | undefined, 283 393 writeResult: string | undefined, 284 394 ): Promise<BenchResult> => { 285 - const tinybench = createTinybench(options).add(name, fn, fnOpts) 286 - const tasks = await runBenchmarks(tinybench) 287 - const task = tinybench.getTask(name)! 288 - if (task.result.state === 'errored') { 289 - throw task.result.error 395 + const results = await runGroup([{ name, fn, fnOpts }], options) 396 + const result = results.get(name) 397 + if (!result) { 398 + throw new Error(`benchmark provider did not return a result for "${name}"`) 290 399 } 291 - await recordBenchmark(tasks, tinybench.name, meta ? new Map([[name, meta]]) : undefined) 400 + await recordBenchmark([result], groupName(options), meta ? new Map([[name, meta]]) : undefined) 292 401 if (writeResult) { 293 - await writeResultArtifact(writeResult, task.result as BenchResult) 402 + await writeResultArtifact(writeResult, result) 294 403 } 295 - return task.result as BenchResult 404 + return result 296 405 } 297 406 298 407 const runFrom = async ( ··· 309 418 return data as BenchResult 310 419 } 311 420 312 - const bench: Bench = (nameOrFunction: string | Function, a: Fn | BenchFnOptions, b?: Fn | BenchFnOptions) => { 421 + const bench: Bench = (nameOrFunction: string | Function, a: BenchFn | BenchFnOptions, b?: BenchFn | BenchFnOptions) => { 313 422 validateBenchmarkProject(config) 314 423 const { fn, fnOpts, writeResult, perProject } = normalizeBenchArgs(a, b) 315 424 const name = typeof nameOrFunction === 'function' ? nameOrFunction.name || '<anonymous>' : nameOrFunction ··· 319 428 name, 320 429 fn, 321 430 fnOpts, 322 - run: (options?: BenchCompareOptions) => { 431 + run: (options?: BenchRunOptions) => { 323 432 pending.delete(registration) 324 433 return runSingle(name, fn, fnOpts, options, meta, writeResult) 325 434 }, ··· 359 468 bench.compare = async (...args) => { 360 469 validateBenchmarkProject(config) 361 470 362 - // extract optional trailing BenchCompareOptions argument 471 + // extract optional trailing BenchRunOptions argument 363 472 const lastArg = args.at(-1) 364 473 const isOptions = lastArg != null && typeof lastArg === 'object' && !(kRegistration in lastArg) 365 - const benchOptions = isOptions ? args.pop() as BenchCompareOptions : undefined 474 + const benchOptions = isOptions ? args.pop() as BenchRunOptions : undefined 366 475 const registrations = args as BenchRegistration<any>[] 367 476 368 477 // Mark every passed-in registration as consumed before validation so a ··· 416 525 } 417 526 } 418 527 419 - const tinybench = createTinybench(benchOptions) 420 - runnable.forEach((reg) => { 421 - tinybench.add(reg.name, reg.fn, reg.fnOpts) 422 - }) 423 - 424 - let tasks: TinybenchTask[] = [] 528 + let results = new Map<string, BenchResult>() 425 529 if (runnable.length > 0) { 426 - tasks = await runBenchmarks(tinybench) 427 - const errors = tinybench.tasks 428 - .filter(task => task.result.state === 'errored') 429 - .map(task => (task.result as any).error) 430 - if (errors.length > 0) { 431 - throw new AggregateError(errors, 'Some benchmarks failed') 432 - } 530 + results = await runGroup( 531 + runnable.map(reg => ({ name: reg.name, fn: reg.fn, fnOpts: reg.fnOpts })), 532 + benchOptions, 533 + ) 433 534 } 434 535 435 - await recordBenchmark(tasks, tinybench.name, taskMeta, fromTasks) 536 + await recordBenchmark(Array.from(results.values()), groupName(benchOptions), taskMeta, fromTasks) 436 537 437 538 // write artifacts for every runnable registration that requested it. We 438 539 // do this after recording so a write failure can't be confused with a ··· 441 542 runnable 442 543 .filter(reg => reg[kWriteResult] != null) 443 544 .map((reg) => { 444 - const task = tinybench.getTask(reg.name)! 445 - return writeResultArtifact(reg[kWriteResult]!, task.result as BenchResult) 545 + const result = results.get(reg.name)! 546 + return writeResultArtifact(reg[kWriteResult]!, result) 446 547 }), 447 548 ) 448 549 449 - return createCompareStorage(tinybench, fromResults) 550 + return createCompareStorage(results, fromResults) 450 551 } 451 552 452 553 bench[kFinalize] = () => { ··· 477 578 } 478 579 479 580 function normalizeBenchArgs( 480 - a: Fn | BenchFnOptions, 481 - b: Fn | BenchFnOptions | undefined, 482 - ): { fn: Fn; fnOpts: FnOptions | undefined; writeResult: string | undefined; perProject: boolean } { 581 + a: BenchFn | BenchFnOptions, 582 + b: BenchFn | BenchFnOptions | undefined, 583 + ): { fn: BenchFn; fnOpts: BenchHooks | undefined; writeResult: string | undefined; perProject: boolean } { 483 584 if (typeof a === 'function') { 484 585 if (b !== undefined) { 485 586 throw new TypeError('`bench()` does not accept options as the third argument. Pass options as the second argument instead: `bench(name, options, fn)`.') ··· 491 592 } 492 593 // Strip vitest-specific fields only when present so we don't allocate a new 493 594 // object — preserving referential identity matters: users inspect 494 - // `registration.fnOpts` and tinybench's `add` sees the same object the 495 - // caller passed in. 595 + // `registration.fnOpts` and the provider sees the same object the caller 596 + // passed in. 496 597 if (a.writeResult === undefined && a.perProject === undefined) { 497 - return { fn: b, fnOpts: a as FnOptions, writeResult: undefined, perProject: false } 598 + return { fn: b, fnOpts: a as BenchHooks, writeResult: undefined, perProject: false } 498 599 } 499 600 const { writeResult, perProject, ...fnOpts } = a 500 601 return { 501 602 fn: b, 502 - fnOpts: Object.keys(fnOpts).length > 0 ? fnOpts as FnOptions : undefined, 603 + fnOpts: Object.keys(fnOpts).length > 0 ? fnOpts as BenchHooks : undefined, 503 604 writeResult, 504 605 perProject: perProject ?? false, 505 606 }

+56

packages/vitest/src/runtime/benchmark/default-provider.ts

··· 1 + import type { Task as TinybenchTask } from 'tinybench' 2 + import type { BenchmarkGroup, BenchmarkProvider, BenchResult } from '../benchmark' 3 + import type { SerializedConfig } from '../config' 4 + import { Bench as Tinybench } from 'tinybench' 5 + import { TestRunner } from '../runners/test' 6 + 7 + const now = globalThis.performance 8 + ? globalThis.performance.now.bind(globalThis.performance) 9 + : Date.now 10 + 11 + /** 12 + * The built-in benchmark provider, backed by tinybench. Selected when 13 + * `benchmark.provider` is `'default'` (the default). 14 + */ 15 + export function createDefaultBenchmarkProvider(config: SerializedConfig): BenchmarkProvider { 16 + let benchIdx = 0 17 + return { 18 + async run({ test, options, registrations }: BenchmarkGroup): Promise<BenchResult[]> { 19 + const currentIndex = ++benchIdx 20 + const tinybench = new Tinybench({ 21 + signal: test.context.signal, 22 + name: `${test.fullTestName} ${currentIndex}`, 23 + retainSamples: config.benchmark.retainSamples, 24 + ...options, 25 + now, 26 + }) 27 + for (const { name, fn, fnOpts } of registrations) { 28 + tinybench.add(name, fn, fnOpts) 29 + } 30 + await TestRunner.runBenchmarks(tinybench) 31 + 32 + const errors = tinybench.tasks 33 + .filter(task => task.result.state === 'errored') 34 + .map(task => (task.result as { error: unknown }).error) 35 + if (errors.length === 1) { 36 + throw errors[0] 37 + } 38 + if (errors.length > 1) { 39 + throw new AggregateError(errors, 'Some benchmarks failed') 40 + } 41 + 42 + return tinybench.tasks.map(toBenchResult) 43 + }, 44 + } 45 + } 46 + 47 + function toBenchResult(task: TinybenchTask): BenchResult { 48 + const result = task.result 49 + if (result.state !== 'completed') { 50 + throw new Error(`task "${task.name}" did not complete: received "${result.state}"`) 51 + } 52 + return { 53 + ...result, 54 + name: task.name, 55 + } 56 + }

+1

packages/vitest/src/runtime/config.ts

··· 143 143 benchmark: { 144 144 enabled: boolean 145 145 retainSamples: boolean 146 + provider: string 146 147 suppressExportGetterWarnings: boolean 147 148 projectName: string 148 149 }

+4 -1

packages/vitest/src/runtime/runners/test.ts

··· 240 240 let _bench: Bench | undefined 241 241 const runnerConfig = this.config 242 242 const benchInstances = this.benchInstances 243 + const moduleRunner = this.moduleRunner 243 244 Object.defineProperty(context, 'bench', { 244 245 get() { 245 246 if (!_bench) { 246 - _bench = createBench(context.task, runnerConfig) 247 + _bench = createBench(context.task, runnerConfig, { 248 + import: id => moduleRunner.import(id), 249 + }) 247 250 benchInstances.set(context.task, _bench) 248 251 } 249 252 return _bench

+115 -1

test/e2e/test/benchmarking.test.ts

··· 1131 1131 expect(stderr).toMatchInlineSnapshot(` 1132 1132 "stderr | getter-warning.bench.ts > getter warning 1133 1133 Benchmark Warning 1134 - Benchmark "getter warning 1" accessed module export getters too many times. 1134 + Benchmark "getter warning" accessed module export getters too many times. 1135 1135 1136 1136 This can make results unreliable because export getters add overhead. 1137 1137 See https://vitest.dev/guide/benchmarking#module-runner-overhead ··· 1320 1320 .toThrow(TypeError) 1321 1321 expect(() => expect({ foo: 'bar' }).toBeSlowerThan(fakeResult(1.0))) 1322 1322 .toThrow(TypeError) 1323 + }) 1324 + 1325 + test('`benchmark.provider` runs a custom provider whose returned results are authoritative', async () => { 1326 + const tasks: TestBenchmarkTask[] = [] 1327 + const { stderr } = await runInlineTests( 1328 + { 1329 + // A provider that never runs the benchmark functions. It returns 1330 + // fabricated statistics keyed by registration name so the assertions 1331 + // below can only pass if Vitest reports exactly what the provider 1332 + // returned, not anything measured off tinybench. 1333 + 'my-provider.ts': /* ts */` 1334 + function stats(mean) { 1335 + return { 1336 + aad: 0, critical: 0, df: 0, mad: 0, max: mean, samples: undefined, 1337 + mean, min: mean, moe: 0, p50: mean, p75: mean, p99: mean, 1338 + p995: mean, p999: mean, rme: 0, samplesCount: 1, sd: 0, sem: 0, variance: 0, 1339 + } 1340 + } 1341 + export default { 1342 + getProvider() { 1343 + return { 1344 + async run({ registrations }) { 1345 + return registrations.map((reg, i) => ({ 1346 + name: reg.name, 1347 + state: 'completed', 1348 + latency: stats((i + 1) * 100), 1349 + throughput: stats(1 / ((i + 1) * 100)), 1350 + period: (i + 1) * 100, 1351 + totalTime: (i + 1) * 1000, 1352 + })) 1353 + }, 1354 + } 1355 + }, 1356 + } 1357 + `, 1358 + 'custom.bench.ts': /* ts */` 1359 + import { test } from 'vitest' 1360 + test('custom provider', async ({ bench }) => { 1361 + await bench.compare( 1362 + bench('first', () => { throw new Error('provider must not call fn') }), 1363 + bench('second', () => { throw new Error('provider must not call fn') }), 1364 + ) 1365 + }) 1366 + `, 1367 + }, 1368 + { 1369 + benchmark: { enabled: true, provider: './my-provider.ts' }, 1370 + reporters: [{ 1371 + onTestCaseBenchmark(_tc, benchmark) { 1372 + tasks.push(...benchmark.tasks) 1373 + }, 1374 + }], 1375 + }, 1376 + ) 1377 + 1378 + expect(stderr).toBe('') 1379 + // the benchmark fns throw — a passing run proves the provider never invoked 1380 + // them, and the reported numbers are the provider's fabricated values 1381 + const byName = Object.fromEntries(tasks.map(t => [t.name, t.latency.mean])) 1382 + expect(byName).toEqual({ first: 100, second: 200 }) 1383 + // ranking is applied by Vitest over the provider's results 1384 + expect(tasks.map(t => ({ name: t.name, rank: t.rank })).sort((a, b) => a.rank - b.rank)) 1385 + .toEqual([{ name: 'first', rank: 1 }, { name: 'second', rank: 2 }]) 1386 + }) 1387 + 1388 + test('`benchmark.provider` receives the registrations with raw fn and fnOpts', async () => { 1389 + // A provider that actually runs each fn once, so we can prove the raw 1390 + // benchmark function and its lifecycle hooks reach the provider untouched. 1391 + const { stderr, results } = await runInlineTests( 1392 + { 1393 + 'runner-provider.ts': /* ts */` 1394 + function stats(mean) { 1395 + return { 1396 + aad: 0, critical: 0, df: 0, mad: 0, max: mean, samples: undefined, 1397 + mean, min: mean, moe: 0, p50: mean, p75: mean, p99: mean, 1398 + p995: mean, p999: mean, rme: 0, samplesCount: 1, sd: 0, sem: 0, variance: 0, 1399 + } 1400 + } 1401 + export default { 1402 + getProvider() { 1403 + return { 1404 + async run({ registrations }) { 1405 + const out = [] 1406 + for (const reg of registrations) { 1407 + await reg.fnOpts?.beforeAll?.() 1408 + await reg.fn() 1409 + await reg.fnOpts?.afterAll?.() 1410 + out.push({ 1411 + name: reg.name, state: 'completed', 1412 + latency: stats(1), throughput: stats(1), period: 1, totalTime: 1, 1413 + }) 1414 + } 1415 + return out 1416 + }, 1417 + } 1418 + }, 1419 + } 1420 + `, 1421 + 'raw.bench.ts': /* ts */` 1422 + import { test, expect } from 'vitest' 1423 + test('raw registration', async ({ bench }) => { 1424 + const calls = [] 1425 + await bench('x', { 1426 + beforeAll: () => { calls.push('beforeAll') }, 1427 + afterAll: () => { calls.push('afterAll') }, 1428 + }, () => { calls.push('fn') }).run() 1429 + expect(calls).toEqual(['beforeAll', 'fn', 'afterAll']) 1430 + }) 1431 + `, 1432 + }, 1433 + { benchmark: { enabled: true, provider: './runner-provider.ts' } }, 1434 + ) 1435 + expect(stderr).toBe('') 1436 + expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed') 1323 1437 }) 1324 1438 1325 1439 declare module 'vitest' {

Configure Feed

Configure Feed