[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
feat: Implement error persistence and live updated for components
+103
-48
+18
-7
src/backend/common/AbstractComponent.ts
+18
-7
src/backend/common/AbstractComponent.ts
···
37
37
import { SourceType } from "./infrastructure/config/source/sources.js";
38
38
import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.js";
39
39
import dayjs from "dayjs";
40
-
import { COMPONENT_STATE, ComponentCommonApi, ComponentState, PlayApiCommonDetailed } from "../../core/Api.js";
40
+
import { COMPONENT_STATE, ComponentCommonApi, ComponentCommonApiJson, ComponentState, PlayApiCommonDetailed } from "../../core/Api.js";
41
41
import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
42
42
import { MarkRequired } from "ts-essentials";
43
+
import { serializeError } from "serialize-error";
43
44
44
45
export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
45
46
···
223
224
await repo.retentionCleanup(this.componentType, this.retentionOpts);
224
225
this.setStatus('Retention cleanup finished');
225
226
} catch (e) {
226
-
this.logger.warn(new Error('Failed to do retention cleanup', {cause: e}));
227
+
const retentionErr = new Error('Failed to do retention cleanup', {cause: e});
228
+
this.warning = retentionErr;
229
+
this.logger.warn(retentionErr);
227
230
this.setStatus('Retention cleanup failed');
228
231
}
229
232
}
···
522
525
523
526
public abstract getRunningState(): ComponentState
524
527
525
-
public getApiData(): Omit<ComponentMinimalSelect, 'type' | 'countLive'> & Pick<ComponentCommonApi, 'state'> {
528
+
public getApiData(): Omit<ComponentCommonApiJson, 'type' | 'countLive' | 'players'> & Pick<ComponentCommonApi, 'state' | 'error' | 'warning'> {
526
529
let state: ComponentState;
527
530
if(!this.initializedOnce || this.initializing) {
528
531
state = COMPONENT_STATE.INITIALIZING;
···
538
541
state,
539
542
mode: this.dbComponent.mode,
540
543
countNonLive: this.dbComponent.countNonLive,
541
-
createdAt: this.dbComponent.createdAt,
542
-
lastReadyAt: this.dbComponent.lastReadyAt,
543
-
lastActiveAt: this.dbComponent.lastActiveAt,
544
+
createdAt: this.dbComponent.createdAt?.toISOString(),
545
+
lastReadyAt: this.dbComponent.lastReadyAt?.toISOString(),
546
+
lastActiveAt: this.dbComponent.lastActiveAt?.toISOString(),
547
+
error: this.error !== undefined && this.error instanceof Error ? serializeError(this.error) : this.error,
548
+
warning: this.warning !== undefined && this.warning instanceof Error ? serializeError(this.warning) : this.warning,
544
549
...this.additionalApiData()
545
550
}
546
551
}
···
555
560
});
556
561
}
557
562
558
-
protected emitComponentUpdate = <T = Partial<typeof this.getApiData>>(payload: T) => {
563
+
protected emitComponentUpdate = <T extends Partial<ReturnType<typeof this.getApiData>>>(payload: T) => {
564
+
if('error' in payload && payload.error instanceof Error) {
565
+
payload.error = serializeError(payload.error);
566
+
}
567
+
if('warning' in payload && payload.warning instanceof Error) {
568
+
payload.warning = serializeError(payload.warning);
569
+
}
559
570
this.emitEvent('componentUpdate', payload);
560
571
}
561
572
protected emitPlayUpdate = (payload: MarkRequired<Partial<PlayApiCommonDetailed>, 'uid'>) => {
+9
-3
src/backend/common/AbstractInitializable.ts
+9
-3
src/backend/common/AbstractInitializable.ts
···
1
-
import { childLogger, Logger } from "@foxxmd/logging";
1
+
import { Logger } from "@foxxmd/logging";
2
2
import {truncateStringToLength } from "../../core/StringUtils.js";
3
3
import { hasNodeNetworkException } from "./errors/NodeErrors.js";
4
4
import { hasUpstreamError } from "./errors/UpstreamError.js";
5
5
import { WebhookPayload } from "./infrastructure/config/health/webhooks.js";
6
-
import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError, TransformRulesError } from "./errors/MSErrors.js";
6
+
import { AuthCheckError, BuildDataError, ConnectionCheckError, ParseCacheError, PostInitError, StageError } from "./errors/MSErrors.js";
7
7
import { messageWithCausesTruncatedDefault } from "../../core/ErrorUtils.js";
8
8
import { spawn } from 'abort-controller-x';
9
9
···
19
19
databaseOK?: boolean | null;
20
20
connectionOK?: boolean | null;
21
21
cacheOK?: boolean | null;
22
+
error?: Error;
23
+
warning?: Error;
22
24
23
25
protected initializedOnce: boolean = false;
24
26
initializing: boolean = false;
···
86
88
} catch (e) {
87
89
throw new PostInitError('Error occurred during post-initialization hook', {cause: e});
88
90
}
91
+
this.error = undefined;
92
+
this.warning = undefined;
89
93
return true;
90
94
} catch(e) {
91
95
if(notify) {
92
96
await this.notify({identifier: this.getIdentifier(), title: notifyTitle, message: truncateStringToLength(500)(messageWithCausesTruncatedDefault(e)), priority: 'error'});
93
97
}
94
-
throw new Error('Initialization failed', {cause: e});
98
+
const initError = new Error('Initialization failed', {cause: e});
99
+
this.error = initError;
100
+
throw initError;
95
101
} finally {
96
102
this.initializing = false;
97
103
}
+5
-1
src/backend/common/database/drizzle/repositories/PlayRepository.ts
+5
-1
src/backend/common/database/drizzle/repositories/PlayRepository.ts
···
254
254
const where = buildPlayWhere({componentId: args.componentId ?? this.componentId, ...rest});
255
255
256
256
// https://github.com/drizzle-team/drizzle-orm/issues/5218#issuecomment-3854900241
257
-
const filter = relationsFilterToSQL(plays, where, relations.plays.relations, relations);
257
+
const filter = relationsFilterToSQL(plays,
258
+
// @ts-expect-error don't have a good way to type this yet
259
+
where,
260
+
relations.plays.relations,
261
+
relations);
258
262
// https://github.com/drizzle-team/drizzle-orm/discussions/3119#discussioncomment-16379557
259
263
const count = await this.db.$count(plays, filter);
260
264
+34
-19
src/backend/scrobblers/AbstractScrobbleClient.ts
+34
-19
src/backend/scrobblers/AbstractScrobbleClient.ts
···
254
254
'Heartbeat',
255
255
(): Promise<any> => {
256
256
return this.heartbeatTask().then(() => null).catch((err) => {
257
+
this.error = err;
257
258
this.logger.error(err);
258
259
});
259
260
},
260
261
(err: Error) => {
262
+
this.error = err;
261
263
this.logger.error(err);
262
264
}
263
265
), {id: 'heartbeat'}));
···
281
283
(): Promise<any> => {
282
284
if(this.isReady()) {
283
285
return this.processDeadLetterQueue().then(() => null).catch((e) => {
286
+
this.warning = e;
284
287
this.logger.error(e);
285
288
})
286
289
}
287
290
return new Promise((resolve, reject) => resolve);
288
291
},
289
292
(err: Error) => {
293
+
this.warning = err;
290
294
this.logger.error(err);
291
295
}
292
296
), {id: 'dead'}));
···
426
430
}
427
431
}
428
432
429
-
public getApiData(): ComponentClientApi {
433
+
public getApiData(): ComponentClientApiJson {
430
434
return {
431
435
...super.getApiData(),
432
436
...this.getComponentApiData(),
···
501
505
const t = new AsyncTask('Playing Now', (): Promise<any> => {
502
506
return this.processingPlayingNow();
503
507
}, (err: Error) => {
504
-
this.npLogger.error(new Error('Unexpected error while processing Now Playing queue', {cause: err}));
508
+
const npErr = new Error('Unexpected error while processing Now Playing queue', {cause: err});
509
+
this.npLogger.error(npErr);
510
+
this.warning = npErr;
511
+
this.emitComponentUpdate<Partial<ComponentClientApiJson>>({warning: npErr});
505
512
});
506
513
507
514
// even though we are processing every 5 seconds the interval that Now Playing is updated at, and that the queue is cleared on,
···
806
813
}
807
814
}
808
815
} catch (e) {
809
-
this.logger.warn(new SimpleError('Could not preload scrobbles', {cause: e, shortStack: true}));
816
+
const preloadErr = new SimpleError('Could not preload scrobbles', {cause: e, shortStack: true});
817
+
this.warning = preloadErr;
818
+
this.emitComponentUpdate<Partial<ComponentClientApiJson>>({warning: preloadErr});
819
+
this.logger.warn(preloadErr);
810
820
}
811
821
}
812
822
}
···
973
983
974
984
await this.startScrobbling(signal);
975
985
}).catch((e) => {
976
-
let status: string;
986
+
const componentUpdate: Partial<ComponentClientApiJson> = {
987
+
state: COMPONENT_STATE.IDLE
988
+
};
977
989
if (isAbortError(e)) {
978
990
const err = generateLoggableAbortReason('Scrobble processing stopped', this.scrobbleQueueAbortController.signal);
979
991
this.logger.info(err);
980
992
this.logger.trace(e);
981
-
status = 'Processing cancelled';
993
+
componentUpdate.status = 'Processing cancelled';
982
994
} else {
983
-
this.logger.warn(new Error('Scrobble processing stopped with error', { cause: e }));
984
-
status = 'Processing stopped with error';
995
+
const err = new Error('Scrobble processing stopped with error', { cause: e });
996
+
this.logger.warn(err);
997
+
componentUpdate.status = 'Processing stopped with error';
998
+
componentUpdate.warning = err;
985
999
}
986
-
this.emitComponentUpdate<Partial<ComponentClientApiJson>>({state: COMPONENT_STATE.IDLE, status});
1000
+
this.emitComponentUpdate<Partial<ComponentClientApiJson>>(componentUpdate);
987
1001
}).finally(() => {
988
1002
this.scrobbleQueueAbortController = undefined;
989
1003
this.scrobbleQueuePromise = undefined;
···
1021
1035
if(!this.isUsable()) {
1022
1036
this.logger.warn('Stopping scrobble processing due to client no longer usable.');
1023
1037
await this.notify({title: `Processing Error`, message: `Encountered error while scrobble processing and client is no longer usable, stopping processing!. | Error: ${e.message}`, priority: 'error'});
1024
-
break;
1038
+
throw e;
1025
1039
} else if (this.authGated()) {
1026
1040
this.logger.warn('Stopping scrobble processing due to client no longer being authenticated.');
1027
1041
await this.notify({title: ` Processing Error`, message: `Encountered error while scrobble processing and client is no longer authenticated, stopping processing!. | Error: ${e.message}`, priority: 'error'});
1028
-
break;
1042
+
throw e;
1029
1043
} else if (this.scrobbleRetries < maxRetries) {
1030
1044
const delayFor = pollingBackoff(this.scrobbleRetries + 1, retryMultiplier);
1031
1045
this.logger.info(`Scrobble processing retries (${this.scrobbleRetries}) less than max processing retries (${maxRetries}), restarting processing after ${delayFor} second delay...`);
···
1034
1048
} else {
1035
1049
this.logger.warn(`Scrobble processing retries (${this.scrobbleRetries}) equal to max processing retries (${maxRetries}), stopping processing!`);
1036
1050
await this.notify({title: `Processing Error`, message: `Encountered error while scrobble processing and retries (${this.scrobbleRetries}) are equal to max processing retries (${maxRetries}), stopping processing!. | Error: ${e.message}`, priority: 'error'});
1051
+
throw e;
1037
1052
}
1038
1053
this.scrobbleRetries++;
1039
1054
}
···
1067
1082
signal.throwIfAborted();
1068
1083
this.logger.info('Scrobble processing started');
1069
1084
this.emitEvent('statusChange', {status: 'Running'});
1085
+
this.emitComponentUpdate<Partial<ComponentClientApiJson>>({state: COMPONENT_STATE.RUNNING});
1070
1086
1071
1087
try {
1072
1088
this.setStatus('Waiting for Plays from Sources');
···
1081
1097
if(nextQueued !== undefined) {
1082
1098
while (nextQueued !== undefined) {
1083
1099
await this.processQueueCurrentScrobble(nextQueued, signal);
1100
+
if(this.error !== undefined) {
1101
+
// we made it through a scrobble without any issues so clear any issue we may have previously had
1102
+
this.error = undefined;
1103
+
this.emitComponentUpdate<Partial<ComponentClientApiJson>>({error: null});
1104
+
}
1084
1105
nextQueued = await this.playRepo.getQueueNext(CLIENT_INGRESS_QUEUE)
1085
1106
}
1086
1107
this.emitEvent('queueEmptied', {});
···
1095
1116
this.logger.error(e);
1096
1117
}
1097
1118
this.emitEvent('statusChange', {status: 'Idle'});
1119
+
this.emitComponentUpdate<Partial<ComponentClientApiJson>>({state: COMPONENT_STATE.IDLE});
1098
1120
this.scrobbling = false;
1099
1121
throw e;
1100
1122
}
···
1825
1847
}
1826
1848
}
1827
1849
1828
-
export const nowPlayingUpdateByPlayDuration: NowPlayingUpdateThreshold = (play?: PlayObject) => {
1829
-
if(play === undefined) {
1830
-
31;
1831
-
}
1832
-
return (play?.data?.duration ?? 30) + 1;
1833
-
}
1850
+
export const nowPlayingUpdateByPlayDuration: NowPlayingUpdateThreshold = (play?: PlayObject) => (play?.data?.duration ?? 30) + 1
1834
1851
1835
-
export const shouldClearNPStatus = (data: SourcePlayerObj) => {
1836
-
return [CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused].includes(data.status.calculated as ReportedPlayerStatus);
1837
-
}
1852
+
export const shouldClearNPStatus = (data: SourcePlayerObj) => [CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused].includes(data.status.calculated as ReportedPlayerStatus)
1838
1853
1839
1854
export const playerInNPPlayingOnlyState = (data: SourcePlayerObj): [boolean, string] => {
1840
1855
// for lower-interval update clients (like listenbrainz, lastfm) IE not real-time
+3
-3
src/backend/server/api.ts
+3
-3
src/backend/server/api.ts
···
41
41
import { playSelectToDeadScrobble } from "../common/database/drizzle/entityUtils.js";
42
42
import AbstractHistoricalScrobbleClient from "../scrobblers/AbstractHistoricalScrobbleClient.js";
43
43
import { DrizzlePlayHistoricalRepository } from "../common/database/drizzle/repositories/PlayHistoricalRepository.js";
44
-
import { ComponentClientApi, ComponentSourceApi, ComponentSourceApiJson } from "../../core/Api.js";
44
+
import { ComponentClientApi, ComponentClientApiJson, ComponentSourceApi, ComponentSourceApiJson } from "../../core/Api.js";
45
45
import { asDayjsHydratedObject } from "../../core/DataUtils.js";
46
46
import { Dayjs } from "dayjs";
47
47
import { asSerializablePlaySelect } from "../../core/PlayMarshalUtils.js";
···
206
206
requiresAuthInteraction = false,
207
207
authed = false
208
208
} = x;
209
-
const base: ComponentSourceApi = x.getApiData();
209
+
const base: ComponentSourceApiJson = x.getApiData();
210
210
if(!x.isReady()) {
211
211
if(x.buildOK === false) {
212
212
base.status = 'Initializing Data Failed';
···
235
235
authed = false,
236
236
scrobbling = false,
237
237
} = x;
238
-
const base: ComponentClientApi = x.getApiData();
238
+
const base: ComponentClientApiJson = x.getApiData();
239
239
240
240
if (!x.isReady()) {
241
241
if(x.buildOK === false) {
+21
-11
src/backend/sources/AbstractSource.ts
+21
-11
src/backend/sources/AbstractSource.ts
···
159
159
'Heartbeat',
160
160
(): Promise<any> => {
161
161
return this.heartbeatTask().then(() => null).catch((err) => {
162
+
this.error = err;
162
163
this.logger.error(err);
163
164
});
164
165
},
165
166
(err: Error) => {
166
167
this.logger.error(err);
168
+
this.error = err;
167
169
}
168
170
), {id: 'heartbeat'}));
169
171
} else {
···
182
184
await this.initialize({force: false, notify: true, notifyTitle: 'Could not initialize automatically'});
183
185
} catch (e) {
184
186
this.logger.error(new Error('Could not initialize automatically', {cause: e}));
185
-
this.setStatus('Could not initialzie automatically');
187
+
this.setStatus('Could not initialize automatically');
186
188
return false;
187
189
}
188
190
···
286
288
}
287
289
}
288
290
289
-
public getApiData(): ComponentSourceApi {
291
+
public getApiData(): ComponentSourceApiJson {
290
292
return {
291
293
...super.getApiData(),
292
294
...this.getComponentApiData(),
···
528
530
try {
529
531
await this.initialize(options);
530
532
} catch (e) {
531
-
this.logger.error(new Error('Cannot start polling because Source is not ready', {cause: e}));
533
+
const err = new Error('Cannot start polling because Source is not ready', {cause: e});
534
+
this.logger.error(err);
532
535
this.setStatus('Polling Error');
536
+
this.emitComponentUpdate<Partial<ComponentSourceApiJson>>({error: err});
537
+
this.error = err;
533
538
if(notify) {
534
539
await this.notify( {title: `Polling Error`, message: `Cannot start polling because Source is not ready: ${truncateStringToLength(500)(messageWithCausesTruncatedDefault(e))}`, priority: 'error'});
535
540
}
···
569
574
});
570
575
await this.startPolling(signal);
571
576
}).catch((e) => {
572
-
let status: string;
577
+
const componentUpdate: Partial<ComponentSourceApiJson> = {
578
+
state: COMPONENT_STATE.IDLE
579
+
};
573
580
if (isAbortError(e)) {
574
581
const err = generateLoggableAbortReason('Polling stopped', this.abortController.signal);
575
582
this.logger.info(err);
576
583
this.logger.trace(e)
577
-
status = 'Polling cancelled';
584
+
componentUpdate.status = 'Polling cancelled';
578
585
} else {
579
-
this.logger.warn(new Error('Polling stopped with error', { cause: e }));
580
-
status = 'Polling stopped with error';
586
+
const err = new Error('Polling stopped with error', { cause: e });
587
+
this.logger.warn(err);
588
+
componentUpdate.status = 'Polling stopped with error';
589
+
componentUpdate.warning = err;
590
+
this.warning = err;
581
591
}
582
-
this.emitComponentUpdate<Partial<ComponentSourceApiJson>>({state: COMPONENT_STATE.IDLE, status});
592
+
this.emitComponentUpdate<Partial<ComponentSourceApiJson>>(componentUpdate);
583
593
}).finally(() => {
584
594
this.abortController = undefined;
585
595
this.pollingPromise = undefined;
···
764
774
this.logger[lastActivityLogLevel](activityMsgs.join(' | '));
765
775
this.setWakeAt(pollFrom.add(sleepTime, 'seconds'));
766
776
this.setIsSleeping(true);
767
-
this.emitComponentUpdate({sleeping: true, wakeAt: this.getWakeAt().toISOString()})
777
+
this.emitComponentUpdate<Partial<ComponentSourceApiJson>>({sleeping: true, wakeAt: this.getWakeAt().toISOString()})
768
778
// set last active before we sleep
769
779
this.componentRepo.updateById(this.dbComponent.id, {lastActiveAt: dayjs(), lastReadyAt: dayjs()});
770
780
while(dayjs().isBefore(this.getWakeAt())) {
···
772
782
await delay(signal, 500);
773
783
}
774
784
this.setIsSleeping(false);
775
-
this.emitComponentUpdate({sleeping: false});
785
+
this.emitComponentUpdate<Partial<ComponentSourceApiJson>>({sleeping: false});
776
786
// if we have made it this far in the loop we can reset poll retries
777
787
this.pollRetries = 0;
778
788
}
···
787
797
throw e;
788
798
} finally {
789
799
this.setIsSleeping(false);
790
-
this.emitComponentUpdate({sleeping: false});
800
+
this.emitComponentUpdate<Partial<ComponentSourceApiJson>>({sleeping: false});
791
801
}
792
802
}
793
803
-1
src/backend/sources/DeezerSource.ts
-1
src/backend/sources/DeezerSource.ts
+2
-2
src/backend/sources/MemorySource.ts
+2
-2
src/backend/sources/MemorySource.ts
···
29
29
import { GenericPlayerState } from "./PlayerState/GenericPlayerState.js";
30
30
import { hashObject } from "../utils/StringUtils.js";
31
31
import { useDebugValue } from "react";
32
-
import { ComponentSourceApi } from "../../core/Api.js";
32
+
import { ComponentSourceApi, ComponentSourceApiJson } from "../../core/Api.js";
33
33
34
34
const EXPECTED_NON_DISCOVERED_REASON = 'not added because an identical play with the same timestamp was already discovered.';
35
35
···
179
179
return record;
180
180
}
181
181
182
-
public getApiData(): ComponentSourceApi {
182
+
public getApiData(): ComponentSourceApiJson {
183
183
return {
184
184
...super.getApiData(),
185
185
sot: this.playerSourceOfTruth,
+9
-1
src/client/components/msComponent/MSComponentDetailed.tsx
+9
-1
src/client/components/msComponent/MSComponentDetailed.tsx
···
147
147
148
148
export const ComponentDetailedDesktop = (props: {data?: ComponentCommonApiJson, live?: boolean}) => {
149
149
let sleepingRender: React.JSX.Element = null;
150
-
const {data} = props;
150
+
const {
151
+
data,
152
+
data: {
153
+
warning,
154
+
error
155
+
} = {}
156
+
} = props;
151
157
const isSource = isComponentSourceApiJson(data)
152
158
if(isSource) {
153
159
const {
···
186
192
<Flex justifyContent="flex-end" rowGap="6" flexDirection="row-reverse" wrap="wrap">
187
193
<Box marginEnd="auto"><MSComponentStats {...props}/></Box>
188
194
</Flex>
195
+
{error !== undefined && error !== null ? <ErrorAlert error={error}/> : undefined}
196
+
{warning !== undefined && warning !== null ? <ErrorAlert error={warning} status="warning"/> : undefined}
189
197
<MSErrorBoundary>{props.live ? <PlayersContainerFetchable nowPlaying={isSource ? undefined : true} data={props.data}/> : <PlayersContainer nowPlaying={isSource ? undefined : true} data={props.data} live={props.live}/>}</MSErrorBoundary>
190
198
<Heading size="3xl" width="100%">{isComponentTypeSource(props.data.mode) ? 'Plays' : 'Scrobbles'}</Heading>
191
199
<MSErrorBoundary><ListContainerFilterable render="virtDynamic" componentType={props.data.mode} componentId={props.data.id}/></MSErrorBoundary>
+2
src/core/Api.ts
+2
src/core/Api.ts
···
81
81
/** More specific, live activity state like "sleeping", "hydrating historical scrobbles", "processing dead scrobbles", etc... */
82
82
status?: string
83
83
players: Record<string, SourcePlayerJson>
84
+
error?: ErrorIsh
85
+
warning?: ErrorIsh
84
86
} & Omit<ComponentMinimalSelect, 'type'>
85
87
86
88
export type ComponentCommonApiJson = Replace<ComponentCommonApi, PickKeys<ComponentCommonApi, Dayjs>, string>;