perf: improve performance of forks pool (#5592) · vitest.dev/vitest@d8304bb

+15 -4

pnpm-lock.yaml

··· 879 879 version: 4.3.10 880 880 debug: 881 881 specifier: ^4.3.4 882 - version: 4.3.4(supports-color@8.1.1) 882 + version: 4.3.4 883 883 execa: 884 884 specifier: ^8.0.1 885 885 version: 8.0.1 ··· 7064 7064 resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==} 7065 7065 engines: {node: '>= 14'} 7066 7066 dependencies: 7067 - debug: 4.3.4(supports-color@8.1.1) 7067 + debug: 4.3.4 7068 7068 transitivePeerDependencies: 7069 7069 - supports-color 7070 7070 dev: true ··· 8569 8569 ms: 2.1.3 8570 8570 supports-color: 8.1.1 8571 8571 dev: true 8572 + 8573 + /debug@4.3.4: 8574 + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} 8575 + engines: {node: '>=6.0'} 8576 + peerDependencies: 8577 + supports-color: '*' 8578 + peerDependenciesMeta: 8579 + supports-color: 8580 + optional: true 8581 + dependencies: 8582 + ms: 2.1.2 8572 8583 8573 8584 /debug@4.3.4(supports-color@8.1.1): 8574 8585 resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} ··· 10855 10866 engines: {node: '>= 14'} 10856 10867 dependencies: 10857 10868 agent-base: 7.1.0 10858 - debug: 4.3.4(supports-color@8.1.1) 10869 + debug: 4.3.4 10859 10870 transitivePeerDependencies: 10860 10871 - supports-color 10861 10872 dev: true ··· 10906 10917 engines: {node: '>= 14'} 10907 10918 dependencies: 10908 10919 agent-base: 7.1.0 10909 - debug: 4.3.4(supports-color@8.1.1) 10920 + debug: 4.3.4 10910 10921 transitivePeerDependencies: 10911 10922 - supports-color 10912 10923 dev: true

+25 -4

packages/vite-node/src/server.ts

··· 31 31 web: new Map<string, Promise<TransformResult | null | undefined>>(), 32 32 } 33 33 34 + private durations = { 35 + ssr: new Map<string, number[]>(), 36 + web: new Map<string, number[]>(), 37 + } 38 + 34 39 private existingOptimizedDeps = new Set<string>() 35 40 36 41 fetchCaches = { ··· 102 107 return shouldExternalize(id, this.options.deps, this.externalizeCache) 103 108 } 104 109 110 + public getTotalDuration() { 111 + const ssrDurations = [...this.durations.ssr.values()].flat() 112 + const webDurations = [...this.durations.web.values()].flat() 113 + return [...ssrDurations, ...webDurations].reduce((a, b) => a + b, 0) 114 + } 115 + 105 116 private async ensureExists(id: string): Promise<boolean> { 106 117 if (this.existingOptimizedDeps.has(id)) 107 118 return true ··· 138 149 } 139 150 140 151 async fetchModule(id: string, transformMode?: 'web' | 'ssr'): Promise<FetchResult> { 141 - const moduleId = normalizeModuleId(id) 142 152 const mode = transformMode || this.getTransformMode(id) 153 + return this.fetchResult(id, mode) 154 + .then((r) => { 155 + return this.options.sourcemap !== true ? { ...r, map: undefined } : r 156 + }) 157 + } 158 + 159 + async fetchResult(id: string, mode: 'web' | 'ssr') { 160 + const moduleId = normalizeModuleId(id) 143 161 this.assertMode(mode) 144 162 const promiseMap = this.fetchPromiseMap[mode] 145 163 // reuse transform for concurrent requests 146 164 if (!promiseMap.has(moduleId)) { 147 165 promiseMap.set(moduleId, this._fetchModule(moduleId, mode) 148 - .then((r) => { 149 - return this.options.sourcemap !== true ? { ...r, map: undefined } : r 150 - }) 151 166 .finally(() => { 152 167 promiseMap.delete(moduleId) 153 168 })) ··· 267 282 timestamp: time, 268 283 result, 269 284 } 285 + 286 + const durations = this.durations[transformMode].get(filePath) || [] 287 + this.durations[transformMode].set( 288 + filePath, 289 + [...durations, duration ?? 0], 290 + ) 270 291 271 292 this.fetchCaches[transformMode].set(filePath, cacheEntry) 272 293 this.fetchCache.set(filePath, cacheEntry)

+8 -2

packages/web-worker/src/utils.ts

··· 1 + import { readFileSync } from 'node:fs' 1 2 import type { WorkerGlobalState } from 'vitest' 2 3 import ponyfillStructuredClone from '@ungap/structured-clone' 3 4 import createDebug from 'debug' ··· 65 66 const { config, rpc, mockMap, moduleCache } = state 66 67 67 68 return { 68 - fetchModule(id: string) { 69 - return rpc.fetch(id, 'web') 69 + async fetchModule(id: string) { 70 + const result = await rpc.fetch(id, 'web') 71 + if (result.id && !result.externalize) { 72 + const code = readFileSync(result.id, 'utf-8') 73 + return { code } 74 + } 75 + return result 70 76 }, 71 77 resolveId(id: string, importer?: string) { 72 78 return rpc.resolveId(id, importer, 'web')

+14 -1

packages/vitest/src/node/workspace.ts

··· 1 1 import { promises as fs } from 'node:fs' 2 + import { rm } from 'node:fs/promises' 3 + import { tmpdir } from 'node:os' 2 4 import fg from 'fast-glob' 3 5 import mm from 'micromatch' 4 6 import { dirname, isAbsolute, join, relative, resolve, toNamespacedPath } from 'pathe' ··· 8 10 import c from 'picocolors' 9 11 import { createBrowserServer } from '../integrations/browser/server' 10 12 import type { ProvidedContext, ResolvedConfig, UserConfig, UserWorkspaceConfig, Vitest } from '../types' 11 - import { deepMerge } from '../utils' 12 13 import type { Typechecker } from '../typecheck/typechecker' 13 14 import type { BrowserProvider } from '../types/browser' 14 15 import { getBrowserProvider } from '../integrations/browser' 16 + import { deepMerge, nanoid } from '../utils/base' 15 17 import { isBrowserEnabled, resolveConfig } from './config' 16 18 import { WorkspaceVitestPlugin } from './plugins/workspace' 17 19 import { createViteServer } from './vite' ··· 77 79 } | undefined 78 80 79 81 testFilesList: string[] | null = null 82 + 83 + public readonly id = nanoid() 84 + public readonly tmpDir = join(tmpdir(), this.id) 80 85 81 86 private _globalSetups: GlobalSetupFile[] | undefined 82 87 private _provided: ProvidedContext = {} as any ··· 402 407 this.server.close(), 403 408 this.typechecker?.stop(), 404 409 this.browser?.close(), 410 + this.clearTmpDir(), 405 411 ].filter(Boolean)).then(() => this._provided = {} as any) 406 412 } 407 413 return this.closingPromise 414 + } 415 + 416 + private async clearTmpDir() { 417 + try { 418 + await rm(this.tmpDir, { force: true, recursive: true }) 419 + } 420 + catch {} 408 421 } 409 422 410 423 async initBrowserProvider() {

+7 -1

packages/vitest/src/runtime/execute.ts

··· 1 1 import vm from 'node:vm' 2 2 import { pathToFileURL } from 'node:url' 3 + import { readFileSync } from 'node:fs' 3 4 import type { ModuleCacheMap } from 'vite-node/client' 4 5 import { DEFAULT_REQUEST_STUBS, ViteNodeRunner } from 'vite-node/client' 5 6 import { isInternalRequest, isNodeBuiltin, isPrimitive, toFilePath } from 'vite-node/utils' ··· 104 105 return { externalize: id } 105 106 } 106 107 107 - return rpc().fetch(id, getTransformMode()) 108 + const result = await rpc().fetch(id, getTransformMode()) 109 + if (result.id && !result.externalize) { 110 + const code = readFileSync(result.id, 'utf-8') 111 + return { code } 112 + } 113 + return result 108 114 }, 109 115 resolveId(id, importer) { 110 116 return rpc().resolveId(id, importer, getTransformMode())

+4 -1

packages/vitest/src/types/rpc.ts

··· 9 9 type TransformMode = 'web' | 'ssr' 10 10 11 11 export interface RuntimeRPC { 12 - fetch: (id: string, environment: TransformMode) => Promise<FetchResult> 12 + fetch: (id: string, environment: TransformMode) => Promise<{ 13 + externalize?: string 14 + id?: string 15 + }> 13 16 transform: (id: string, environment: TransformMode) => Promise<FetchResult> 14 17 resolveId: (id: string, importer: string | undefined, environment: TransformMode) => Promise<ViteNodeResolveId | null> 15 18 getSourceMap: (id: string, force?: boolean) => Promise<RawSourceMap | undefined>

+11

packages/vitest/src/utils/base.ts

··· 169 169 export function wildcardPatternToRegExp(pattern: string): RegExp { 170 170 return new RegExp(`^${pattern.split('*').map(escapeRegExp).join('.*')}$`, 'i') 171 171 } 172 + 173 + // port from nanoid 174 + // https://github.com/ai/nanoid 175 + const urlAlphabet 176 + = 'useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict' 177 + export function nanoid(size = 21) { 178 + let id = '' 179 + let i = size 180 + while (i--) id += urlAlphabet[(Math.random() * 64) | 0] 181 + return id 182 + }

+7 -1

packages/vitest/src/integrations/env/loader.ts

··· 1 + import { readFileSync } from 'node:fs' 1 2 import { normalize, resolve } from 'pathe' 2 3 import { ViteNodeRunner } from 'vite-node/client' 3 4 import type { ViteNodeRunnerOptions } from 'vite-node' ··· 26 27 return environments[name] 27 28 const loader = await createEnvironmentLoader({ 28 29 root: ctx.config.root, 29 - fetchModule: id => rpc.fetch(id, 'ssr'), 30 + fetchModule: async (id) => { 31 + const result = await rpc.fetch(id, 'ssr') 32 + if (result.id) 33 + return { code: readFileSync(result.id, 'utf-8') } 34 + return result 35 + }, 30 36 resolveId: (id, importer) => rpc.resolveId(id, importer, 'ssr'), 31 37 }) 32 38 const root = loader.root

+30 -2

packages/vitest/src/node/pools/rpc.ts

··· 1 + import { mkdir, writeFile } from 'node:fs/promises' 1 2 import type { RawSourceMap } from 'vite-node' 3 + import { join } from 'pathe' 2 4 import type { RuntimeRPC } from '../../types' 3 5 import type { WorkspaceProject } from '../workspace' 6 + 7 + const created = new Set() 8 + const promises = new Map<string, Promise<void>>() 4 9 5 10 export function createMethodsRPC(project: WorkspaceProject): RuntimeRPC { 6 11 const ctx = project.ctx ··· 20 25 const r = await project.vitenode.transformRequest(id) 21 26 return r?.map as RawSourceMap | undefined 22 27 }, 23 - fetch(id, transformMode) { 24 - return project.vitenode.fetchModule(id, transformMode) 28 + async fetch(id, transformMode) { 29 + const result = await project.vitenode.fetchResult(id, transformMode) 30 + const code = result.code 31 + if (result.externalize) 32 + return result 33 + if ('id' in result) 34 + return { id: result.id as string } 35 + 36 + if (!code) 37 + throw new Error(`Failed to fetch module ${id}`) 38 + 39 + const dir = join(project.tmpDir, transformMode) 40 + const tmp = join(dir, id.replace(/[/\\?%*:|"<>]/g, '_').replace('\0', '__x00__')) 41 + if (promises.has(tmp)) { 42 + await promises.get(tmp) 43 + return { id: tmp } 44 + } 45 + if (!created.has(dir)) { 46 + await mkdir(dir, { recursive: true }) 47 + created.add(dir) 48 + } 49 + promises.set(tmp, writeFile(tmp, code, 'utf-8').finally(() => promises.delete(tmp))) 50 + await promises.get(tmp) 51 + Object.assign(result, { id: tmp }) 52 + return { id: tmp } 25 53 }, 26 54 resolveId(id, importer, transformMode) { 27 55 return project.vitenode.resolveId(id, importer, transformMode)

+1 -1

packages/vitest/src/node/reporters/base.ts

··· 232 232 const setupTime = files.reduce((acc, test) => acc + Math.max(0, test.setupDuration || 0), 0) 233 233 const testsTime = files.reduce((acc, test) => acc + Math.max(0, test.result?.duration || 0), 0) 234 234 const transformTime = this.ctx.projects 235 - .flatMap(w => Array.from(w.vitenode.fetchCache.values()).map(i => i.duration || 0)) 235 + .flatMap(w => w.vitenode.getTotalDuration()) 236 236 .reduce((a, b) => a + b, 0) 237 237 const environmentTime = files.reduce((acc, file) => acc + Math.max(0, file.environmentLoad || 0), 0) 238 238 const prepareTime = files.reduce((acc, file) => acc + Math.max(0, file.prepareDuration || 0), 0)

Configure Feed

Configure Feed