[READ-ONLY] Mirror of https://github.com/vitest-dev/vitest. Next generation testing framework powered by Vite.
vitest.dev
test
testing-tools
vite
chore: temporary adaptive-scaling benchmark workflow (drop before merge)
Vladimir Sheremet
(Jul 9, 2026, 10:03 AM +0200)
7d4bb6e4
3c4e4590
+166
+122
scripts/bench-adaptive.mjs
+122
scripts/bench-adaptive.mjs
···
1
+
// TEMPORARY (PR-only): A/B benchmark for the pool's adaptive worker scaling.
2
+
// Generates small fixtures inside the repo (so `vitest` resolves from the
3
+
// workspace root), then interleaves reps of VITEST_POOL_ADAPTIVE=1 (default)
4
+
// vs VITEST_POOL_ADAPTIVE=0 (previous behavior) within every cell.
5
+
import { spawnSync } from 'node:child_process'
6
+
import { mkdirSync, rmSync, writeFileSync } from 'node:fs'
7
+
import { join } from 'node:path'
8
+
9
+
const REPO = new URL('..', import.meta.url).pathname.replace(/^\/(\w:)/, '$1')
10
+
const CLI = join(REPO, 'packages/vitest/dist/cli.js')
11
+
const REPS = Number(process.env.BENCH_REPS || 5)
12
+
13
+
const FIXTURE_ROOT = join(REPO, '.bench-adaptive')
14
+
15
+
function generateFixture(name, fileCount) {
16
+
const root = join(FIXTURE_ROOT, name)
17
+
rmSync(root, { recursive: true, force: true })
18
+
mkdirSync(join(root, 'test'), { recursive: true })
19
+
mkdirSync(join(root, 'src'), { recursive: true })
20
+
for (let i = 0; i < 5; i++) {
21
+
writeFileSync(join(root, `src/m${i}.ts`), `
22
+
export function m${i}(x: number): number {
23
+
return x * ${i + 2}
24
+
}
25
+
`)
26
+
}
27
+
for (let t = 0; t < fileCount; t++) {
28
+
writeFileSync(join(root, `test/t${t}.test.ts`), `
29
+
import { expect, test } from 'vitest'
30
+
import { m${t % 5} } from '../src/m${t % 5}'
31
+
32
+
test('t${t} computes', () => {
33
+
expect(m${t % 5}(${t})).toBe(${t * ((t % 5) + 2)})
34
+
})
35
+
test('t${t} strings', () => {
36
+
expect('t${t}'.repeat(2)).toContain('t${t}')
37
+
})
38
+
`)
39
+
}
40
+
writeFileSync(join(root, 'vitest.config.ts'), `
41
+
import { defineConfig } from 'vitest/config'
42
+
43
+
export default defineConfig({
44
+
cacheDir: 'node_modules/.vite-bench',
45
+
test: {
46
+
watch: false,
47
+
environment: 'node',
48
+
pool: process.env.BENCH_POOL as 'forks',
49
+
isolate: process.env.BENCH_ISOLATE !== '0',
50
+
reporters: [['default', { summary: false }]],
51
+
},
52
+
})
53
+
`)
54
+
return root
55
+
}
56
+
57
+
function runOnce(root, pool, isolate, adaptive) {
58
+
const env = {
59
+
...process.env,
60
+
NO_COLOR: '1',
61
+
CI: 'true',
62
+
BENCH_POOL: pool,
63
+
BENCH_ISOLATE: isolate ? '1' : '0',
64
+
VITEST_POOL_ADAPTIVE: adaptive ? '1' : '0',
65
+
}
66
+
const start = performance.now()
67
+
const result = spawnSync(process.execPath, [CLI, 'run', '--root', root], {
68
+
env,
69
+
encoding: 'utf-8',
70
+
maxBuffer: 64 * 1024 * 1024,
71
+
})
72
+
const wall = performance.now() - start
73
+
if (result.status !== 0) {
74
+
console.error(`FAILED pool=${pool} isolate=${isolate} adaptive=${adaptive}`)
75
+
console.error(result.stdout?.slice(-2000))
76
+
console.error(result.stderr?.slice(-2000))
77
+
process.exit(1)
78
+
}
79
+
return wall
80
+
}
81
+
82
+
function median(values) {
83
+
const sorted = [...values].sort((a, b) => a - b)
84
+
return sorted[sorted.length >> 1]
85
+
}
86
+
87
+
const small = generateFixture('small', 40)
88
+
const large = generateFixture('large', 160)
89
+
90
+
const CELLS = [
91
+
{ label: 'forks isolate:false 40 files', root: small, pool: 'forks', isolate: false },
92
+
{ label: 'forks isolate:true 40 files', root: small, pool: 'forks', isolate: true },
93
+
{ label: 'forks isolate:false 160 files', root: large, pool: 'forks', isolate: false },
94
+
{ label: 'threads isolate:false 40 files', root: small, pool: 'threads', isolate: false },
95
+
]
96
+
97
+
const rows = []
98
+
for (const cell of CELLS) {
99
+
// warm up the transform cache once so the reps measure scheduling, not
100
+
// first-transform noise
101
+
runOnce(cell.root, cell.pool, cell.isolate, true)
102
+
const adaptive = []
103
+
const fixed = []
104
+
for (let rep = 0; rep < REPS; rep++) {
105
+
adaptive.push(runOnce(cell.root, cell.pool, cell.isolate, true))
106
+
fixed.push(runOnce(cell.root, cell.pool, cell.isolate, false))
107
+
console.error(`${cell.label} rep${rep}: adaptive=${adaptive.at(-1) | 0}ms fixed=${fixed.at(-1) | 0}ms`)
108
+
}
109
+
const adaptiveMed = median(adaptive)
110
+
const fixedMed = median(fixed)
111
+
const delta = ((adaptiveMed - fixedMed) / fixedMed * 100).toFixed(1)
112
+
rows.push(`| ${cell.label} | ${fixedMed | 0}ms | ${adaptiveMed | 0}ms | ${delta}% |`)
113
+
}
114
+
115
+
console.log('\n### Adaptive pool scaling A/B (median of %d, lower is better)\n', REPS)
116
+
console.log('| cell | fixed (old) | adaptive | Δ |')
117
+
console.log('|---|---:|---:|---:|')
118
+
for (const row of rows) {
119
+
console.log(row)
120
+
}
121
+
122
+
rmSync(FIXTURE_ROOT, { recursive: true, force: true })
+44
.github/workflows/bench-adaptive.yml
+44
.github/workflows/bench-adaptive.yml
···
1
+
# TEMPORARY (PR-only, removed before merge): measures the pool's adaptive
2
+
# worker scaling on/off on the platforms where worker spawn costs differ the
3
+
# most. Windows is the interesting one — process spawn is several times more
4
+
# expensive there.
5
+
name: bench-adaptive
6
+
7
+
on:
8
+
pull_request:
9
+
paths:
10
+
- '.github/workflows/bench-adaptive.yml'
11
+
- 'scripts/bench-adaptive.mjs'
12
+
- 'packages/vitest/src/node/pools/**'
13
+
workflow_dispatch:
14
+
15
+
permissions: {}
16
+
17
+
jobs:
18
+
bench:
19
+
name: 'Bench: adaptive scaling, ${{ matrix.os }}'
20
+
runs-on: ${{ matrix.os }}
21
+
timeout-minutes: 40
22
+
23
+
strategy:
24
+
matrix:
25
+
os: [windows-latest, ubuntu-latest]
26
+
fail-fast: false
27
+
28
+
steps:
29
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
30
+
with:
31
+
persist-credentials: false
32
+
33
+
- uses: ./.github/actions/setup-and-cache
34
+
with:
35
+
node-version: 24
36
+
37
+
- name: Install
38
+
run: pnpm i --filter '!docs'
39
+
40
+
- name: Build
41
+
run: pnpm run build
42
+
43
+
- name: Bench
44
+
run: node scripts/bench-adaptive.mjs