feat(coverage): `thresholds` to support maximum uncovered items (#7061) · vitest.dev/vitest@bde98b6

+21 -6

docs/config/index.md

··· 1481 1481 1482 1482 #### coverage.thresholds 1483 1483 1484 - Options for coverage thresholds 1484 + Options for coverage thresholds. 1485 + 1486 + If a threshold is set to a positive number, it will be interpreted as the minimum percentage of coverage required. For example, setting the lines threshold to `90` means that 90% of lines must be covered. 1487 + 1488 + If a threshold is set to a negative number, it will be treated as the maximum number of uncovered items allowed. For example, setting the lines threshold to `-10` means that no more than 10 lines may be uncovered. 1489 + 1490 +  1491 + ```ts 1492 + { 1493 + coverage: { 1494 + thresholds: { 1495 + // Requires 90% function coverage 1496 + functions: 90, 1497 + 1498 + // Require that no more than 10 lines are uncovered 1499 + lines: -10, 1500 + } 1501 + } 1502 + } 1503 + ``` 1485 1504 1486 1505 ##### coverage.thresholds.lines 1487 1506 ··· 1490 1509 - **CLI:** `--coverage.thresholds.lines=<number>` 1491 1510 1492 1511 Global threshold for lines. 1493 - See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. 1494 1512 1495 1513 ##### coverage.thresholds.functions 1496 1514 ··· 1499 1517 - **CLI:** `--coverage.thresholds.functions=<number>` 1500 1518 1501 1519 Global threshold for functions. 1502 - See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. 1503 1520 1504 1521 ##### coverage.thresholds.branches 1505 1522 ··· 1508 1525 - **CLI:** `--coverage.thresholds.branches=<number>` 1509 1526 1510 1527 Global threshold for branches. 1511 - See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. 1512 1528 1513 1529 ##### coverage.thresholds.statements 1514 1530 ··· 1517 1533 - **CLI:** `--coverage.thresholds.statements=<number>` 1518 1534 1519 1535 Global threshold for statements. 1520 - See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. 1521 1536 1522 1537 ##### coverage.thresholds.perFile 1523 1538 ··· 1535 1550 - **Available for providers:** `'v8' | 'istanbul'` 1536 1551 - **CLI:** `--coverage.thresholds.autoUpdate=<boolean>` 1537 1552 1538 - Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds. 1553 + Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is better than the configured thresholds. 1539 1554 This option helps to maintain thresholds when coverage is improved. 1540 1555 1541 1556 ##### coverage.thresholds.100

+10 -10

test/coverage-test/test/threshold-auto-update.test.ts

··· 20 20 // Global ones 21 21 lines: 0.1, 22 22 functions: 0.2, 23 - branches: 0.3, 24 - statements: 0.4, 23 + branches: -1000, 24 + statements: -2000, 25 25 26 26 '**/src/math.ts': { 27 27 branches: 0.1, 28 28 functions: 0.2, 29 - lines: 0.3, 30 - statements: 0.4 29 + lines: -1000, 30 + statements: -2000, 31 31 } 32 32 } 33 33 } ··· 56 56 lines: 55.55, 57 57 functions: 33.33, 58 58 branches: 100, 59 - statements: 55.55, 59 + statements: -8, 60 60 61 61 '**/src/math.ts': { 62 62 branches: 100, 63 63 functions: 25, 64 - lines: 50, 65 - statements: 50 64 + lines: -6, 65 + statements: -6, 66 66 } 67 67 } 68 68 } ··· 84 84 lines: 33.33, 85 85 functions: 33.33, 86 86 branches: 100, 87 - statements: 33.33, 87 + statements: -4, 88 88 89 89 '**/src/math.ts': { 90 90 branches: 100, 91 91 functions: 25, 92 - lines: 25, 93 - statements: 25 92 + lines: -3, 93 + statements: -3, 94 94 } 95 95 } 96 96 }

+31 -1

test/coverage-test/test/threshold-failure.test.ts

··· 2 2 import { sum } from '../fixtures/src/math' 3 3 import { coverageTest, isV8Provider, normalizeURL, runVitest, test } from '../utils' 4 4 5 - test('failing thresholds', async () => { 5 + test('failing percentage thresholds', async () => { 6 6 const { exitCode, stderr } = await runVitest({ 7 7 include: [normalizeURL(import.meta.url)], 8 8 coverage: { ··· 26 26 expect(stderr).toContain(`ERROR: Coverage for lines (${lines}) does not meet "**/fixtures/src/math.ts" threshold (100%)`) 27 27 expect(stderr).toContain(`ERROR: Coverage for statements (${statements}) does not meet "**/fixtures/src/math.ts" threshold (100%)`) 28 28 expect(stderr).toContain('ERROR: Coverage for functions (25%) does not meet "**/fixtures/src/math.ts" threshold (100%)') 29 + }) 30 + 31 + test('failing absolute thresholds', async () => { 32 + const { exitCode, stderr } = await runVitest({ 33 + include: [normalizeURL(import.meta.url)], 34 + coverage: { 35 + all: false, 36 + include: ['**/fixtures/src/math.ts'], 37 + thresholds: { 38 + '**/fixtures/src/math.ts': { 39 + branches: -1, 40 + functions: -2, 41 + lines: -5, 42 + statements: -1, 43 + }, 44 + }, 45 + }, 46 + }, { throwOnError: false }) 47 + 48 + expect(exitCode).toBe(1) 49 + 50 + if (isV8Provider()) { 51 + expect(stderr).toContain('ERROR: Uncovered lines (6) exceed "**/fixtures/src/math.ts" threshold (5)') 52 + expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)') 53 + expect(stderr).toContain('ERROR: Uncovered statements (6) exceed "**/fixtures/src/math.ts" threshold (1)') 54 + } 55 + else { 56 + expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)') 57 + expect(stderr).toContain('ERROR: Uncovered statements (3) exceed "**/fixtures/src/math.ts" threshold (1)') 58 + } 29 59 }) 30 60 31 61 coverageTest('cover some lines, but not too much', () => {

+56 -9

packages/vitest/src/utils/coverage.ts

··· 363 363 for (const thresholdKey of THRESHOLD_KEYS) { 364 364 const threshold = thresholds[thresholdKey] 365 365 366 - if (threshold !== undefined) { 366 + if (threshold === undefined) { 367 + continue 368 + } 369 + 370 + /** 371 + * Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered), 372 + * while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered). 373 + */ 374 + if (threshold >= 0) { 367 375 const coverage = summary.data[thresholdKey].pct 368 376 369 377 if (coverage < threshold) { 370 378 process.exitCode = 1 371 379 372 - /* 380 + /** 373 381 * Generate error message based on perFile flag: 374 382 * - ERROR: Coverage for statements (33.33%) does not meet threshold (85%) for src/math.ts 375 383 * - ERROR: Coverage for statements (50%) does not meet global threshold (85%) 376 384 */ 377 - let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${ 378 - name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` 385 + let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` 379 386 } threshold (${threshold}%)` 387 + 388 + if (this.options.thresholds?.perFile && file) { 389 + errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` 390 + } 391 + 392 + this.ctx.logger.error(errorMessage) 393 + } 394 + } 395 + else { 396 + const uncovered = summary.data[thresholdKey].total - summary.data[thresholdKey].covered 397 + const absoluteThreshold = threshold * -1 398 + 399 + if (uncovered > absoluteThreshold) { 400 + process.exitCode = 1 401 + 402 + /** 403 + * Generate error message based on perFile flag: 404 + * - ERROR: Uncovered statements (33) exceed threshold (30) for src/math.ts 405 + * - ERROR: Uncovered statements (33) exceed global threshold (30) 406 + */ 407 + let errorMessage = `ERROR: Uncovered ${thresholdKey} (${uncovered}) exceed ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` 408 + } threshold (${absoluteThreshold})` 380 409 381 410 if (this.options.thresholds?.perFile && file) { 382 411 errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` ··· 416 445 417 446 for (const key of THRESHOLD_KEYS) { 418 447 const threshold = thresholds[key] ?? 100 419 - const actual = Math.min( 420 - ...summaries.map(summary => summary[key].pct), 421 - ) 448 + /** 449 + * Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered), 450 + * while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered). 451 + */ 452 + if (threshold >= 0) { 453 + const actual = Math.min( 454 + ...summaries.map(summary => summary[key].pct), 455 + ) 422 456 423 - if (actual > threshold) { 424 - thresholdsToUpdate.push([key, actual]) 457 + if (actual > threshold) { 458 + thresholdsToUpdate.push([key, actual]) 459 + } 460 + } 461 + else { 462 + const absoluteThreshold = threshold * -1 463 + const actual = Math.max( 464 + ...summaries.map(summary => summary[key].total - summary[key].covered), 465 + ) 466 + 467 + if (actual < absoluteThreshold) { 468 + // If everything was covered, set new threshold to 100% (since a threshold of 0 would be considered as 0%) 469 + const updatedThreshold = actual === 0 ? 100 : actual * -1 470 + thresholdsToUpdate.push([key, updatedThreshold]) 471 + } 425 472 } 426 473 } 427 474

+4 -4

test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts

··· 9 9 // Global ones 10 10 lines: 0.1, 11 11 functions: 0.2, 12 - branches: 0.3, 13 - statements: 0.4, 12 + branches: -1000, 13 + statements: -2000, 14 14 15 15 '**/src/math.ts': { 16 16 branches: 0.1, 17 17 functions: 0.2, 18 - lines: 0.3, 19 - statements: 0.4 18 + lines: -1000, 19 + statements: -2000, 20 20 } 21 21 } 22 22 }

Configure Feed

Configure Feed