[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
Merge pull request #493 from FoxxMD/deadScrobbleImprovements
feat: Dead scrobble improvements
+87
-55
+2
-2
src/backend/common/infrastructure/config/client/index.ts
+2
-2
src/backend/common/infrastructure/config/client/index.ts
+34
-33
src/backend/scrobblers/AbstractScrobbleClient.ts
+34
-33
src/backend/scrobblers/AbstractScrobbleClient.ts
···
111
111
nowPlayingTaskInterval: number = 5000;
112
112
npLogger: Logger;
113
113
dupeLogger: Logger;
114
+
deadLogger: Logger;
114
115
115
116
existingScrobble: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise<PlayMatchResult>
116
117
···
137
138
this.logger = childLogger(logger, this.getIdentifier());
138
139
this.npLogger = childLogger(this.logger, 'Now Playing');
139
140
this.dupeLogger = childLogger(this.logger, 'Dupe');
141
+
this.deadLogger = childLogger(this.logger, 'Dead');
140
142
this.notifier = notifier;
141
143
this.emitter = emitter;
142
144
this.scrobbledPlayObjs = new FixedSizeList<ScrobbledPlayObject>(this.MAX_STORED_SCROBBLES);
···
753
755
754
756
const {
755
757
options: {
756
-
deadLetterRetries = 1
758
+
deadLetterRetries = 3
757
759
} = {}
758
760
} = this.config;
759
761
···
762
764
const processable = this.deadLetterScrobbles.filter(x => x.retries < retries);
763
765
const queueStatus = `${processable.length} of ${this.deadLetterScrobbles.length} dead scrobbles have less than ${retries} retries, ${processable.length === 0 ? 'will skip processing.': 'processing now...'}`;
764
766
if (processable.length === 0) {
765
-
this.logger.verbose({labels: 'Dead Letter'}, queueStatus);
767
+
this.deadLogger.verbose(queueStatus);
766
768
return;
767
769
}
768
-
this.logger.info({labels: 'Dead Letter'}, queueStatus);
770
+
this.logger.info(queueStatus);
769
771
if(!this.upstreamRefresh.refreshEnabled) {
770
-
this.logger.verbose({labels: 'Dead Letter'}, 'Scrobble refresh is DISABLED. All dead scrobbles will likely always be scrobbled (nothing to check duplicates against).');
772
+
this.deadLogger.verbose('Scrobble refresh is DISABLED. All dead scrobbles will likely always be scrobbled (nothing to check duplicates against).');
771
773
}
772
774
this.handleQueuedScrobbleRanges();
773
775
774
776
const removedIds = [];
775
-
for (const deadScrobble of this.deadLetterScrobbles) {
776
-
if (deadScrobble.retries < retries) {
777
-
const [scrobbled, dead] = await this.processDeadLetterScrobble(deadScrobble.id);
778
-
if (scrobbled) {
779
-
removedIds.push(deadScrobble.id);
780
-
}
777
+
for (const deadScrobble of processable) {
778
+
const [scrobbled, dead] = await this.processDeadLetterScrobble(deadScrobble.id);
779
+
if (scrobbled) {
780
+
removedIds.push(deadScrobble.id);
781
781
}
782
+
await sleep(this.scrobbleSleep);
782
783
}
783
784
if (removedIds.length > 0) {
784
-
this.logger.info({labels: 'Dead Letter'}, `Removed ${removedIds.length} scrobbles from dead letter queue`);
785
+
this.deadLogger.info(`Removed ${removedIds.length} scrobbles from dead letter queue`);
785
786
}
786
787
}
787
788
788
789
processDeadLetterScrobble = async (id: string): Promise<[boolean, DeadLetterScrobble<PlayObject>?]> => {
789
790
const deadScrobbleIndex = this.deadLetterScrobbles.findIndex(x => x.id === id);
791
+
if(deadScrobbleIndex === -1) {
792
+
this.deadLogger.warn(`Could not find a dead scrobble with id ${id}`);
793
+
return [false];
794
+
}
795
+
const deadLabel = {labels: id};
790
796
const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex];
797
+
this.deadLogger.trace(deadLabel, `Processing dead scrobble => ${buildTrackString(deadScrobble.play)}`);
791
798
792
799
if (!(await this.isReady())) {
793
-
this.logger.warn({labels: 'Dead Letter'}, 'Cannot process dead letter scrobble because client is not ready.');
800
+
this.deadLogger.warn(deadLabel, 'Cannot process dead letter scrobble because client is not ready.');
794
801
return [false, deadScrobble];
795
802
}
796
803
let historicalPlays: PlayObject[] = [];
···
799
806
historicalPlays = await this.getSOTScrobblesForPlay(deadScrobble.play);
800
807
} catch (e) {
801
808
if(e.message === 'Cannot get historical plays due to cached error') {
802
-
this.logger.warn(`${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' => Previous error while getting historical scrobbles means this scrobble cannot be compared`);
803
-
this.logger.trace(e);
809
+
this.deadLogger.warn(deadLabel, `Previous error while getting historical scrobbles means this scrobble cannot be compared`);
810
+
this.deadLogger.trace(e);
804
811
} else {
805
-
this.logger.warn(new SimpleError(`${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' => cannot get historical scrobbles`, {cause: e, shortStack: true}));
812
+
this.deadLogger.warn(new SimpleError(`${id} - ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' => cannot get historical scrobbles`, {cause: e, shortStack: true}));
806
813
}
807
814
deadScrobble.retries++;
808
815
deadScrobble.error = messageWithCauses(e);
809
816
deadScrobble.lastRetry = dayjs();
810
817
this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble;
811
818
this.updateDeadLetterCache();
819
+
this.emitEvent('updateDeadLetter', {dead: deadScrobble});
812
820
return [false, deadScrobble];
813
-
} finally {
814
-
await sleep(1000);
815
821
}
816
822
}
817
823
const {summary, ...matchResult} = await this.existingScrobble(deadScrobble.play, historicalPlays);
···
832
838
const scrobbledPlay = await this.scrobble(transformedScrobble);
833
839
this.emitEvent('scrobble', {play: transformedScrobble});
834
840
this.addScrobbledTrack(transformedScrobble, scrobbledPlay);
841
+
this.removeDeadLetterScrobble(deadScrobble.id)
835
842
} catch (e) {
836
843
837
844
const submitError = findCauseByReference(e, ScrobbleSubmitError);
···
847
854
deadScrobble.retries++;
848
855
deadScrobble.error = messageWithCauses(e);
849
856
deadScrobble.lastRetry = dayjs();
850
-
this.logger.error(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.source}' due to error`, {cause: e}));
857
+
this.deadLogger.error(new Error(`${id} - Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.source}' due to error`, {cause: e}));
851
858
this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble;
852
859
this.updateDeadLetterCache();
860
+
this.emitEvent('updateDeadLetter', {dead: deadScrobble});
853
861
return [false, deadScrobble];
854
-
} finally {
855
-
await sleep(1000);
856
862
}
857
-
}
858
-
if(deadScrobble !== undefined) {
863
+
} else {
864
+
this.deadLogger.verbose(`Looks like ${buildTrackString(deadScrobble.play)} was already scrobbled!\n${summary}`);
859
865
this.removeDeadLetterScrobble(deadScrobble.id)
860
866
}
861
867
862
-
return [true];
868
+
return [true, deadScrobble];
863
869
}
864
870
865
871
removeDeadLetterScrobble = (id: string) => {
866
872
const index = this.deadLetterScrobbles.findIndex(x => x.id === id);
867
873
if (index === -1) {
868
-
this.logger.warn(`No scrobble found with ID ${id}`, {leaf: 'Dead Letter'});
874
+
this.deadLogger.warn(`No scrobble found with ID ${id}`);
875
+
return;
869
876
}
870
-
this.logger.info(`Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`, {leaf: 'Dead Letter'});
877
+
this.deadLogger.info({labels: id}, `Removed scrobble ${buildTrackString(this.deadLetterScrobbles[index].play)} from queue`);
871
878
this.deadLetterScrobbles.splice(index, 1);
872
879
this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length);
873
880
this.updateDeadLetterCache();
881
+
this.emitEvent('removeDeadLetter', { dead: { id } });
874
882
}
875
883
876
884
removeDeadLetterScrobbles = () => {
···
878
886
this.updateDeadLetterCache();
879
887
this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterScrobbles.length);
880
888
this.logger.info('Removed all scrobbles from queue', {leaf: 'Dead Letter'});
881
-
}
882
-
883
-
protected getLatestQueuePlayDate = () => {
884
-
if (this.queuedScrobbles.length === 0) {
885
-
return undefined;
886
-
}
887
-
return this.queuedScrobbles[this.queuedScrobbles.length - 1].play.data.playDate;
888
889
}
889
890
890
891
queueScrobble = async (data: PlayObject | PlayObject[], source: string) => {
···
923
924
return (beforeMain + beforeDead) - (this.queuedScrobbles.length + this.deadLetterScrobbles.length);
924
925
}
925
926
926
-
protected addDeadLetterScrobble = (data: QueuedScrobble<PlayObject>, error: (Error | string) = 'Unspecified error') => {
927
+
addDeadLetterScrobble = (data: QueuedScrobble<PlayObject>, error: (Error | string) = 'Unspecified error') => {
927
928
let eString = '';
928
929
if(typeof error === 'string') {
929
930
eString = error;
+3
-5
src/backend/server/api.ts
+3
-5
src/backend/server/api.ts
···
331
331
332
332
(client as AbstractScrobbleClient).logger.verbose('User requested processing of all dead letter scrobbles via API');
333
333
334
-
await (client as AbstractScrobbleClient).processDeadLetterQueue(1000);
335
-
336
-
const result: DeadLetterScrobble<PlayObject>[] = (client as AbstractScrobbleClient).deadLetterScrobbles;
334
+
res.status(200).send('OK');
337
335
338
-
return res.json(result);
336
+
await ((client as AbstractScrobbleClient).processDeadLetterQueue(1000));
339
337
});
340
338
341
339
app.putAsync('/api/dead/:id', clientMiddleFunc(true), async (req, res, next) => {
···
358
356
return res.status(404).send();
359
357
}
360
358
361
-
const [scrobbled, dead] = await (client as AbstractScrobbleClient).processDeadLetterScrobble(deadId);
359
+
const [scrobbled, dead] = await ((client as AbstractScrobbleClient).processDeadLetterScrobble(deadId));
362
360
363
361
if(scrobbled) {
364
362
return res.status(200).send();
+19
src/backend/tests/scrobbler/scrobblers.test.ts
+19
src/backend/tests/scrobbler/scrobblers.test.ts
···
21
21
import { shuffleArray } from '../../utils/DataUtils.js';
22
22
import { DEFAULT_CONSOLIDATE_DURATION, DEFAULT_GROUP_DURATION, groupPlaysToTimeRanges } from '../../utils/ListenFetchUtils.js';
23
23
import { asPlay } from '../../../core/tests/utils/fixtures.js';
24
+
import { nanoid } from 'nanoid';
24
25
25
26
chai.use(asPromised);
26
27
···
592
593
await emptied2;
593
594
scrobbler.tryStopScrobbling().then(() => null);
594
595
expect(sp.calledTwice).is.true;
596
+
});
597
+
598
+
});
599
+
600
+
describe('Dead Scrobbles', function() {
601
+
602
+
it('Processes all dead scrobbles', async function () {
603
+
testScrobbler = generateTestScrobbler();
604
+
await testScrobbler.initialize();
605
+
testScrobbler.testRecentScrobbles = [];
606
+
607
+
const deadPlays = generatePlays(3);
608
+
for(const dead of deadPlays) {
609
+
testScrobbler.addDeadLetterScrobble({source: 'test', play: dead, id: nanoid()});
610
+
}
611
+
await testScrobbler.processDeadLetterQueue();
612
+
await testScrobbler.tryStopScrobbling()
613
+
expect(testScrobbler.deadLetterScrobbles.length).eq(0);
595
614
});
596
615
597
616
});
+29
-15
src/client/deadLetter/deadLetterDucks.ts
+29
-15
src/client/deadLetter/deadLetterDucks.ts
···
92
92
}
93
93
)
94
94
builder.addMatcher(
95
-
(action) => deadApi.endpoints.getDead.matchFulfilled(action) || deadApi.endpoints.processDead.matchFulfilled(action) || deadApi.endpoints.removeDead.matchFulfilled(action),
95
+
(action) => deadApi.endpoints.getDead.matchFulfilled(action) || deadApi.endpoints.removeDead.matchFulfilled(action),
96
96
(state, action) => {
97
97
deadAdapter.setAll(state, action.payload);
98
98
}
99
99
)
100
+
// .addMatcher(
101
+
// (action) => deadApi.endpoints.removeDeadSingle.matchFulfilled(action),
102
+
// (state, action) => {
103
+
// deadAdapter.removeOne(state, action.meta.arg.originalArgs.id)
104
+
// }
105
+
// )
106
+
// .addMatcher(
107
+
// (action) => deadApi.endpoints.processDeadSingle.matchFulfilled(action),
108
+
// (state, action) => {
109
+
// if (action.payload === undefined) {
110
+
// deadAdapter.removeOne(state, action.meta.arg.originalArgs.id);
111
+
// } else {
112
+
// state.entities[action.meta.arg.originalArgs.id] = action.payload;
113
+
// //deadAdapter.updateOne(state, action.meta.arg.originalArgs.id);
114
+
// }
115
+
// }
116
+
// )
100
117
.addMatcher(
101
-
(action) => deadApi.endpoints.removeDeadSingle.matchFulfilled(action),
118
+
(action) => clearDead.match(action),
102
119
(state, action) => {
103
-
deadAdapter.removeOne(state, action.meta.arg.originalArgs.id)
120
+
state = deadAdapter.getInitialState();
104
121
}
105
122
)
106
123
.addMatcher(
107
-
(action) => deadApi.endpoints.processDeadSingle.matchFulfilled(action),
124
+
(action) => clientUpdate.match(action) && action.payload.event === 'deadLetter',
108
125
(state, action) => {
109
-
if (action.payload === undefined) {
110
-
deadAdapter.removeOne(state, action.meta.arg.originalArgs.id);
111
-
} else {
112
-
state.entities[action.meta.arg.originalArgs.id] = action.payload;
113
-
//deadAdapter.updateOne(state, action.meta.arg.originalArgs.id);
114
-
}
126
+
state.entities[(action.payload as ApiEventPayload).data.dead.id] = (action.payload as ApiEventPayload).data.dead;
127
+
state.ids.push((action.payload as ApiEventPayload).data.dead.id);
115
128
}
116
129
)
117
130
.addMatcher(
118
-
(action) => clearDead.match(action),
131
+
(action) => clientUpdate.match(action) && action.payload.event === 'removeDeadLetter',
119
132
(state, action) => {
120
-
state = deadAdapter.getInitialState();
133
+
delete state.entities[(action.payload as ApiEventPayload).data.dead.id];
134
+
state.ids = state.ids.filter(x => x !== (action.payload as ApiEventPayload).data.dead.id);
121
135
}
122
136
)
123
-
/*.addMatcher(
124
-
(action) => clientUpdate.match(action) && action.payload.event === 'deadLetter',
137
+
.addMatcher(
138
+
(action) => clientUpdate.match(action) && action.payload.event === 'updateDeadLetter',
125
139
(state, action) => {
126
140
state.entities[(action.payload as ApiEventPayload).data.dead.id] = (action.payload as ApiEventPayload).data.dead;
127
141
}
128
-
)*/
142
+
)
129
143
}
130
144
});
131
145