+18 -5

docsite/docs/configuration/configuration.mdx

Reviewed

··· 694 694 695 695 ### Health Endpoint 696 696 697 697 - An endpoint for monitoring the health of sources/clients is available at GET `http://YourMultiScrobblerDomain/health` 697 697 + An endpoint for monitoring the health of sources/clients is available at GET `http://YourMultiScrobblerDomain/api/health` 698 698 699 699 * Returns `200 OK` when **everything** is working or `500 Internal Server Error` if **anything** is not 700 700 - * The plain url (`/health`) aggregates status of **all clients/sources** -- so any failing client/source will make status return 500 701 701 - * Use query params `type` or `name` to restrict client/sources aggregated IE `/health?type=spotify` or `/health?name=MyMaloja` 700 700 + * The plain url (`/api/health`) aggregates status of **all clients/sources** -- so any failing client/source will make status return 500 701 701 + * Use query params `type` or `name` to restrict client/sources aggregated IE `/api/health?type=spotify` or `/api/health?name=MyMaloja` 702 702 * On 500 the response returns a JSON payload with `messages` array that describes any issues 703 703 - * For any clients/sources that require authentication `/health` will return 500 if they are **not authenticated** 704 704 - * For sources that poll (spotify, yt music, subsonic) `/health` will 500 if they are **not polling** 703 703 + * For any clients/sources that require authentication `/api/health` will return 500 if they are **not authenticated** 704 704 + * For sources that poll (spotify, yt music, subsonic) `/api/health` will 500 if they are **not polling** 705 705 + 706 706 + ### Prometheus 707 707 + 708 708 + A [Prometheus](https://prometheus.io/) export endpoint is available at GET `http://YourMultiScrobblerDomain/api/metrics` 709 709 + 710 710 + It includes metrics for: 711 711 + 712 712 + * Count of discovered plays per [Source](/configuration/sources) 713 713 + * Count of Queued/Scrobbled/Deadletter scrobbles per [Client](/configuration/clients) 714 714 + * Number of issues per Source/Client 715 715 + * If any of these metrics is > 0 it means your Source/Client is not operating normally 716 716 + 717 717 + Additionally, general process metrics like cpu and memory usage can be enabled with the env `PROMETHEUS_FULL=true`

+38

package-lock.json

Reviewed

··· 84 84 "passport": "^0.6.0", 85 85 "passport-deezer": "^0.2.0", 86 86 "patch-package": "^8.0.0", 87 87 + "prom-client": "^15.1.3", 87 88 "react": "^18.2.0", 88 89 "react-dom": "^18.2.0", 89 90 "react-redux": "^8.1.2", ··· 2212 2213 "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", 2213 2214 "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", 2214 2215 "dev": true 2216 2216 + }, 2217 2217 + "node_modules/@opentelemetry/api": { 2218 2218 + "version": "1.9.0", 2219 2219 + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", 2220 2220 + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", 2221 2221 + "license": "Apache-2.0", 2222 2222 + "engines": { 2223 2223 + "node": ">=8.0.0" 2224 2224 + } 2215 2225 }, 2216 2226 "node_modules/@parcel/watcher": { 2217 2227 "version": "2.4.1", ··· 4570 4580 "funding": { 4571 4581 "url": "https://github.com/sponsors/sindresorhus" 4572 4582 } 4583 4583 + }, 4584 4584 + "node_modules/bintrees": { 4585 4585 + "version": "1.0.2", 4586 4586 + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", 4587 4587 + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", 4588 4588 + "license": "MIT" 4573 4589 }, 4574 4590 "node_modules/body-parser": { 4575 4591 "version": "1.20.3", ··· 10504 10520 "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.0.tgz", 10505 10521 "integrity": "sha512-/MyYDxttz7DfGMMHiysAsFE4qF+pQYAA8ziO/3NcRVrQ5fSk+Mns4QZA/oRPFzvcqNoVJXQNWNAsdwBXLUkQKw==" 10506 10522 }, 10523 10523 + "node_modules/prom-client": { 10524 10524 + "version": "15.1.3", 10525 10525 + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", 10526 10526 + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", 10527 10527 + "license": "Apache-2.0", 10528 10528 + "dependencies": { 10529 10529 + "@opentelemetry/api": "^1.4.0", 10530 10530 + "tdigest": "^0.1.1" 10531 10531 + }, 10532 10532 + "engines": { 10533 10533 + "node": "^16 || ^18 || >=20" 10534 10534 + } 10535 10535 + }, 10507 10536 "node_modules/prop-types": { 10508 10537 "version": "15.8.1", 10509 10538 "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", ··· 11915 11944 "funding": { 11916 11945 "type": "opencollective", 11917 11946 "url": "https://opencollective.com/webpack" 11947 11947 + } 11948 11948 + }, 11949 11949 + "node_modules/tdigest": { 11950 11950 + "version": "0.1.2", 11951 11951 + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", 11952 11952 + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", 11953 11953 + "license": "MIT", 11954 11954 + "dependencies": { 11955 11955 + "bintrees": "1.0.2" 11918 11956 } 11919 11957 }, 11920 11958 "node_modules/test-exclude": {

+1

package.json

Reviewed

··· 116 116 "passport": "^0.6.0", 117 117 "passport-deezer": "^0.2.0", 118 118 "patch-package": "^8.0.0", 119 119 + "prom-client": "^15.1.3", 119 120 "react": "^18.2.0", 120 121 "react-dom": "^18.2.0", 121 122 "react-redux": "^8.1.2",

+32

src/backend/ioc.ts

Reviewed

··· 12 12 import { MSCache } from "./common/Cache.js"; 13 13 import TransformerManager from "./common/transforms/TransformerManager.js"; 14 14 import { TransformerCommonConfig } from "../core/Atomic.js"; 15 15 + import prom, { Counter, Gauge } from 'prom-client'; 15 16 16 17 export let version: string = 'unknown'; 17 18 ··· 33 34 mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap) 34 35 transformers?: TransformerCommonConfig[] 35 36 } 37 37 + 38 38 + const discovered = new prom.Counter({ 39 39 + name: 'multiscrobbler_source_discovered', 40 40 + help: 'Number of discovered plays for a Source', 41 41 + labelNames: ['name', 'type'] 42 42 + }); 43 43 + const queuedGauge = new prom.Gauge({ 44 44 + name: 'multiscrobbler_client_queued', 45 45 + help: 'Number of queued plays for a Client', 46 46 + labelNames: ['name', 'type'] 47 47 + }); 48 48 + const deadLetterGauge = new prom.Gauge({ 49 49 + name: 'multiscrobbler_client_deadletter', 50 50 + help: 'Number of deadletter plays for a Client', 51 51 + labelNames: ['name', 'type'] 52 52 + }); 53 53 + const scrobbledCounter = new prom.Counter({ 54 54 + name: 'multiscrobbler_client_scrobbled', 55 55 + help: 'Number of discovered plays for a Source', 56 56 + labelNames: ['name', 'type'] 57 57 + }); 36 58 37 59 const createRoot = (options: RootOptions = {logger: loggerDebug}) => { 38 60 const { ··· 114 136 notifierEmitter: () => new EventEmitter(), 115 137 loggerStream, 116 138 loggingConfig, 139 139 + sourceMetics: { 140 140 + discovered: discovered, 141 141 + //issues: sourceIssues 142 142 + }, 143 143 + clientMetrics: { 144 144 + queued: queuedGauge, 145 145 + scrobbled: scrobbledCounter, 146 146 + //issues: issuesClientGauge, 147 147 + deadLetter: deadLetterGauge 148 148 + }, 117 149 logger: logger, 118 150 transformerManager, 119 151 cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc,

+28

src/backend/scrobblers/AbstractScrobbleClient.ts

Reviewed

··· 58 58 import { findAsyncSequential, staggerMapper } from "../utils/AsyncUtils.js"; 59 59 import pMap, { pMapIterable } from "p-map"; 60 60 import { comparePlayArtistsNormalized, comparePlayTracksNormalized } from "../utils/PlayComparisonUtils.js"; 61 61 + import { normalizeStr } from "../utils/StringUtils.js"; 62 62 + import prom, { Counter, Gauge } from 'prom-client'; 61 63 62 64 type PlatformMappedPlays = Map<string, {play: PlayObject, source: SourceIdentifier}>; 63 65 type NowPlayingQueue = Map<string, PlatformMappedPlays>; ··· 108 110 109 111 notifier: Notifiers; 110 112 emitter: EventEmitter; 113 113 + 114 114 + protected scrobbledCounter: Counter; 115 115 + protected queuedGauge: Gauge; 116 116 + protected deadLetterGauge: Gauge; 117 117 + protected problemGauge: Gauge; 111 118 112 119 constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) { 113 120 super(config); ··· 160 167 confidenceBreakdown 161 168 } 162 169 }; 170 170 + 171 171 + const clientMetrics = getRoot().items.clientMetrics; 172 172 + this.queuedGauge = clientMetrics.queued; 173 173 + this.deadLetterGauge = clientMetrics.deadLetter; 174 174 + this.scrobbledCounter = clientMetrics.scrobbled; 163 175 } 164 176 165 177 set recentScrobbles(scrobbles: PlayObject[]) { ··· 178 190 protected getMachineId() { 179 191 return `${this.type}-${this.name}`; 180 192 } 193 193 + public getSafeExternalName() { 194 194 + return normalizeStr(this.name, {keepSingleWhitespace: false}); 195 195 + } 196 196 + public getSafeExternalId() { 197 197 + return `${this.type}-${normalizeStr(this.name, {keepSingleWhitespace: false})}`; 198 198 + } 199 199 + 200 200 + protected getPrometheusLabels() { 201 201 + return {name: this.getSafeExternalName(), type: this.type}; 202 202 + } 181 203 182 204 public notify = async (payload: WebhookPayload) => { 183 205 this.emitEvent('notify', payload); ··· 457 479 458 480 addScrobbledTrack = (playObj: PlayObject, scrobbledPlay: PlayObject) => { 459 481 this.scrobbledPlayObjs.add({play: playObj, scrobble: scrobbledPlay}); 482 482 + this.scrobbledCounter.labels(this.getPrometheusLabels()).inc(); 460 483 this.lastScrobbledPlayDate = playObj.data.playDate; 461 484 this.tracksScrobbled++; 462 485 } ··· 791 814 this.logger.debug(`Will not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' because it ${timeFrameValidLog}`); 792 815 } 793 816 this.updateQueuedScrobblesCache(); 817 817 + this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedScrobbles.length); 794 818 this.emitEvent('scrobbleDequeued', {queuedScrobble: currQueuedPlay}) 795 819 } 796 820 await sleep(this.scrobbleSleep); ··· 889 913 } 890 914 this.logger.info(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'}); 891 915 this.deadLetterScrobbles.splice(index, 1); 916 916 + this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length); 892 917 this.updateDeadLetterCache(); 893 918 } 894 919 895 920 removeDeadLetterScrobbles = () => { 896 921 this.deadLetterScrobbles = []; 897 922 this.updateDeadLetterCache(); 923 923 + this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length); 898 924 this.logger.info('Removed all scrobbles from queue', {leaf: 'Dead Letter'}); 899 925 } 900 926 ··· 912 938 const queuedPlay = {id: nanoid(), source, play: play} 913 939 this.emitEvent('scrobbleQueued', {queuedPlay: queuedPlay}); 914 940 this.queuedScrobbles.push(queuedPlay); 941 941 + this.queuedGauge.labels(this.getPrometheusLabels()).inc(); 915 942 // this is wasteful but we don't want the processing loop popping out-of-order (by date) scrobbles 916 943 this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); 917 944 } ··· 929 956 this.deadLetterScrobbles.push(deadData); 930 957 this.deadLetterScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); 931 958 this.emitEvent('deadLetter', {dead: deadData}); 959 959 + this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length); 932 960 this.updateDeadLetterCache(); 933 961 } 934 962

+48

src/backend/server/api.ts

Reviewed

··· 30 30 import { setupWebscrobblerRoutes } from "./webscrobblerRoutes.js"; 31 31 import ScrobbleSources from "../sources/ScrobbleSources.js"; 32 32 import ScrobbleClients from "../scrobblers/ScrobbleClients.js"; 33 33 + import prom from 'prom-client'; 33 34 34 35 const maxBufferSize = 300; 35 36 const output: Record<number, FixedSizeList<LogDataPretty>> = {}; ··· 508 509 509 510 510 511 return res.status((clientsReady && sourcesReady) ? 200 : 500).json({messages: sourceMessages.concat(clientMessages)}); 512 512 + }); 513 513 + 514 514 + const issuesClientGauge = new prom.Gauge({ 515 515 + name: 'multiscrobbler_client_issues', 516 516 + help: 'Number of errors/issues with Client', 517 517 + labelNames: ['name', 'type'], 518 518 + async collect() { 519 519 + for(const client of scrobbleClients.clients) { 520 520 + let issues = 0; 521 521 + if(!(await client.isReady())) { 522 522 + issues++; 523 523 + } 524 524 + this.labels({name: client.getSafeExternalName(), type: client.type}).set(issues); 525 525 + } 526 526 + } 527 527 + }); 528 528 + 529 529 + const sourceIssues = new prom.Gauge({ 530 530 + name: 'multiscrobbler_source_issues', 531 531 + help: 'Number of errors/issues with Source', 532 532 + labelNames: ['name', 'type'], 533 533 + async collect() { 534 534 + for(const source of scrobbleSources.sources) { 535 535 + let issues = 0; 536 536 + if(source.requiresAuth && !source.authed) { 537 537 + issues++; 538 538 + } 539 539 + if(source.canPoll && !source.polling) { 540 540 + issues++; 541 541 + } 542 542 + this.labels({name: source.getSafeExternalName(), type: source.type}).set(issues); 543 543 + } 544 544 + } 545 545 + }); 546 546 + 547 547 + if(process.env.PROMETHEUS_FULL === 'true') { 548 548 + prom.collectDefaultMetrics(); 549 549 + } 550 550 + 551 551 + app.getAsync('/api/metrics', async (req, res) => { 552 552 + 553 553 + const metricsString = await prom.register.metrics(); 554 554 + return res 555 555 + .status(200) 556 556 + .set('Content-Type', 'text/plain') 557 557 + .send(metricsString); 558 558 + 511 559 }); 512 560 513 561 app.getAsync('/api/version', async (req, res) => {

+20 -1

src/backend/sources/AbstractSource.ts

Reviewed

··· 42 42 import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; 43 43 import { findAsync, staggerMapper } from '../utils/AsyncUtils.js'; 44 44 import pMap, {pMapIterable} from 'p-map'; 45 45 - import { randomInt } from 'crypto'; 45 45 + import prom, { Counter, Gauge } from 'prom-client'; 46 46 + import { normalizeStr } from '../utils/StringUtils.js'; 46 47 47 48 export interface RecentlyPlayedOptions { 48 49 limit?: number ··· 91 92 92 93 protected loggerLabel: string; 93 94 95 95 + protected discoveredCounter: Counter; 96 96 + 94 97 constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) { 95 98 super(config); 96 99 const {clients = [] } = config; ··· 105 108 this.localUrl = internal.localUrl; 106 109 this.configDir = internal.configDir; 107 110 this.emitter = emitter; 111 111 + 112 112 + this.discoveredCounter = getRoot().items.sourceMetics.discovered; 108 113 } 109 114 110 115 protected getIdentifier() { 111 116 return `${capitalize(this.type)} - ${this.name}` 112 117 } 118 118 + protected getMachineId() { 119 119 + return `${this.type}-${this.name}`; 120 120 + } 121 121 + public getSafeExternalName() { 122 122 + return normalizeStr(this.name, {keepSingleWhitespace: false}); 123 123 + } 124 124 + public getSafeExternalId() { 125 125 + return `${this.type}-${normalizeStr(this.name, {keepSingleWhitespace: false})}`; 126 126 + } 127 127 + 128 128 + protected getPrometheusLabels() { 129 129 + return {name: this.getSafeExternalName(), type: this.type}; 130 130 + } 113 131 114 132 getSystemListeningBehavior = (): boolean | undefined => { 115 133 if(this.supportsManualListening) { ··· 141 159 this.tracksDiscovered++; 142 160 this.logger.info(`Discovered => ${buildTrackString(play)}`); 143 161 this.emitEvent('discovered', {play}); 162 162 + this.discoveredCounter.labels(this.getPrometheusLabels()).inc(); 144 163 } 145 164 146 165 getFlatRecentlyDiscoveredPlays = (): PlayObject[] =>