[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
feat(database): Initial Souce database usage implementation
+292
-82
+7
-1
src/backend/common/AbstractComponent.ts
+7
-1
src/backend/common/AbstractComponent.ts
···
83
83
protected async doBuildDatabase(): Promise<true | string | undefined> {
84
84
super.doBuildDatabase();
85
85
86
+
let name: string;
87
+
if('name' in this) {
88
+
name = this.name as string;
89
+
}
90
+
86
91
this.dbComponent = await this.componentRepo.findOrInsert({
87
92
mode: this.componentType,
88
93
type: this.type,
89
-
uid: this.config.id ?? this.config.name
94
+
uid: this.config.id ?? this.config.name ?? name,
95
+
name: this.config.name ?? name
90
96
});
91
97
return true;
92
98
}
+14
src/backend/common/AbstractInitializable.ts
+14
src/backend/common/AbstractInitializable.ts
···
198
198
this.databaseOK = false;
199
199
throw new BuildDataError('Required database init failed', {cause: e});
200
200
}
201
+
202
+
try {
203
+
await this.postDatabase();
204
+
} catch (e) {
205
+
if(e instanceof StageError) {
206
+
throw e;
207
+
} else {
208
+
throw new Error('Error occurred during post-database hook', {cause: e});
209
+
}
210
+
}
211
+
}
212
+
213
+
protected async postDatabase(): Promise<void> {
214
+
return;
201
215
}
202
216
203
217
/**
+4
-1
src/backend/common/Cache.ts
+4
-1
src/backend/common/Cache.ts
···
48
48
49
49
cacheMetadata: Cacheable;
50
50
cacheScrobble: Cacheable;
51
+
cacheDb: Cacheable;
51
52
cacheAuth: Cacheable;
52
53
regexCache: ReturnType<typeof cacheFunctions>;
53
54
cacheTransform: Cacheable;
···
114
115
this.cacheAuth = inMemory;
115
116
this.cacheScrobble = inMemory;
116
117
this.cacheApi = inMemory;
118
+
this.cacheDb = new Cacheable({primary: initMemoryCache({lruSize: 500, ttl: '1m'})});
117
119
}
118
120
119
121
init = async (enableCollectors: boolean = false) => {
···
133
135
{ cache: this.cacheScrobble, name: 'queued_scrobbles' },
134
136
{ cache: this.cacheTransform, name: 'transformer' },
135
137
{ cache: this.cacheClientScrobbles, name: 'historical_scrobbles' },
136
-
{ cache: this.cacheApi, name: 'external_apis' }
138
+
{ cache: this.cacheApi, name: 'external_apis' },
139
+
{ cache: this.cacheDb, name: 'database' }
137
140
];
138
141
139
142
this.cacheHits = new prom.Gauge({
+1
-1
src/backend/common/database/drizzle/entityUtils.ts
+1
-1
src/backend/common/database/drizzle/entityUtils.ts
···
15
15
};
16
16
}
17
17
18
-
export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId'>> & { error?: ErrorLike };
18
+
export type PlayEntityOpts = Partial<Pick<PlayNew, 'seenAt' | 'playedAt' | 'uid' | 'state' | 'parentId' | 'componentId' | 'platformId'>> & { error?: ErrorLike };
19
19
20
20
export const generatePlayEntity = (play: PlayObject, opts: PlayEntityOpts = {}): PlayNew => {
21
21
const {
+2
src/backend/common/database/drizzle/migrations/20260426001747_jittery_demogoblin/migration.sql
src/backend/common/database/drizzle/migrations/20260427134808_far_thundra/migration.sql
+2
src/backend/common/database/drizzle/migrations/20260426001747_jittery_demogoblin/migration.sql
src/backend/common/database/drizzle/migrations/20260427134808_far_thundra/migration.sql
···
28
28
`play` text NOT NULL,
29
29
`state` text NOT NULL,
30
30
`parentId` integer,
31
+
`platformId` text,
31
32
`compacted` text,
32
33
CONSTRAINT `fk_plays_componentId_components_id_fk` FOREIGN KEY (`componentId`) REFERENCES `components`(`id`) ON UPDATE CASCADE ON DELETE CASCADE,
33
34
CONSTRAINT `fk_plays_parentId_plays_id_fk` FOREIGN KEY (`parentId`) REFERENCES `plays`(`id`) ON UPDATE CASCADE ON DELETE SET NULL
···
54
55
CREATE UNIQUE INDEX `play_uid_idx` ON `plays` (`uid`);--> statement-breakpoint
55
56
CREATE INDEX `play_playedAt_idx` ON `plays` (`playedAt`);--> statement-breakpoint
56
57
CREATE INDEX `play_seenAt_idx` ON `plays` (`seenAt`);--> statement-breakpoint
58
+
CREATE INDEX `play_platform_idx` ON `plays` (`platformId`);--> statement-breakpoint
57
59
CREATE INDEX `play_queue_state_id_idx` ON `play_queue_states` (`playId`);
+25
-1
src/backend/common/database/drizzle/migrations/20260426001747_jittery_demogoblin/snapshot.json
src/backend/common/database/drizzle/migrations/20260427134808_far_thundra/snapshot.json
+25
-1
src/backend/common/database/drizzle/migrations/20260426001747_jittery_demogoblin/snapshot.json
src/backend/common/database/drizzle/migrations/20260427134808_far_thundra/snapshot.json
···
1
1
{
2
2
"version": "7",
3
3
"dialect": "sqlite",
4
-
"id": "c9d77fe1-7dce-4014-bd6e-10174b7ebccc",
4
+
"id": "2a0aea39-bc8a-436a-a84c-6041c2b63c3f",
5
5
"prevIds": [
6
6
"00000000-0000-0000-0000-000000000000"
7
7
],
···
239
239
"default": null,
240
240
"generated": null,
241
241
"name": "parentId",
242
+
"entityType": "columns",
243
+
"table": "plays"
244
+
},
245
+
{
246
+
"type": "text",
247
+
"notNull": false,
248
+
"autoincrement": false,
249
+
"default": null,
250
+
"generated": null,
251
+
"name": "platformId",
242
252
"entityType": "columns",
243
253
"table": "plays"
244
254
},
···
556
566
"where": null,
557
567
"origin": "manual",
558
568
"name": "play_seenAt_idx",
569
+
"entityType": "indexes",
570
+
"table": "plays"
571
+
},
572
+
{
573
+
"columns": [
574
+
{
575
+
"value": "platformId",
576
+
"isExpression": false
577
+
}
578
+
],
579
+
"isUnique": false,
580
+
"where": null,
581
+
"origin": "manual",
582
+
"name": "play_platform_idx",
559
583
"entityType": "indexes",
560
584
"table": "plays"
561
585
},
+62
-2
src/backend/common/database/drizzle/repositories/PlayRepository.ts
+62
-2
src/backend/common/database/drizzle/repositories/PlayRepository.ts
···
6
6
import { playInputs, plays, relations } from "../schema/schema.js";
7
7
import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey } from "../drizzleTypes.js";;
8
8
import { MarkOptional, MarkRequired, PathValue } from "ts-essentials";
9
-
import { removeUndefinedKeys } from "../../../../utils.js";
9
+
import { genGroupIdStrFromPlay, removeUndefinedKeys } from "../../../../utils.js";
10
10
import dayjs, { Dayjs } from "dayjs";
11
-
import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
11
+
import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and } from "drizzle-orm";
12
12
import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js";
13
13
import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js";
14
14
import { buildDateCompare, CompareDateOp, DrizzleBaseRepository } from "./BaseRepository.js";
···
20
20
}
21
21
export interface PlayWhereOpts {
22
22
state?: PlaySelect['state'][]
23
+
stateNot?: PlaySelect['state'][]
23
24
componentId?: number
25
+
platformId?: string
24
26
seenAt?: CompareDateOp
25
27
playedAt?: CompareDateOp
26
28
}
···
274
276
275
277
loggerCom.verbose(`Cleanup done! Summary:\n${summaryDelStates.join(' | ')}`);
276
278
}
279
+
280
+
public selectRecentDistinctPlatforms = async (componentId: number, limitPlays: number = 500): Promise<string[]> => {
281
+
282
+
const recentPlatformIds = await this.db.selectDistinct({platformId: plays.platformId}).from(plays)
283
+
.where(
284
+
and(
285
+
eq(plays.componentId, componentId),
286
+
ne(plays.state, 'queued'))
287
+
)
288
+
.orderBy(desc(plays.playedAt))
289
+
.limit(limitPlays);
290
+
return recentPlatformIds.map(x => x.platformId);
291
+
}
277
292
}
278
293
279
294
export const buildPlayWhere = (args: PlayWhereOpts): FindWhere<'plays'> => {
···
288
303
in: args.state
289
304
}
290
305
}
306
+
if(args.stateNot !== undefined) {
307
+
where.state = {
308
+
NOT: {
309
+
in: args.stateNot
310
+
}
311
+
}
312
+
}
291
313
if (args.seenAt !== undefined) {
292
314
where.seenAt = buildDateCompare(args.seenAt);
293
315
}
294
316
if (args.playedAt !== undefined) {
295
317
where.playedAt = buildDateCompare(args.playedAt);
296
318
}
319
+
if(args.platformId !== undefined) {
320
+
where.platformId = args.platformId
321
+
}
297
322
return where;
298
323
}
299
324
325
+
export const playToRepositoryCreatePlayOpts = (data: MarkOptional<RepositoryCreatePlayOpts, 'input'>): RepositoryCreatePlayOpts => {
326
+
const {
327
+
play: {
328
+
meta: {
329
+
lifecycle: {
330
+
input,
331
+
original,
332
+
...lifecycleRest
333
+
} = {},
334
+
...metaRest
335
+
},
336
+
...playRest
337
+
},
338
+
...rest
339
+
} = data;
340
+
341
+
return {
342
+
play: {
343
+
...playRest,
344
+
meta: {
345
+
...metaRest,
346
+
// @ts-expect-error
347
+
lifecycle: {
348
+
...lifecycleRest
349
+
}
350
+
}
351
+
},
352
+
...rest,
353
+
input: {
354
+
play: original,
355
+
data: input
356
+
},
357
+
platformId: genGroupIdStrFromPlay(data.play)
358
+
}
359
+
}
+3
-1
src/backend/common/database/drizzle/schema/schema.ts
+3
-1
src/backend/common/database/drizzle/schema/schema.ts
···
29
29
playedAt: DayjsTimestamp('playedAt'),
30
30
seenAt: DayjsTimestamp('seenAt'),
31
31
play: text({ mode: 'json' }).notNull().$type<PlayObject>(),
32
-
state: text({enum: ['queued','discovered','scrobbled','failed','duped']}).notNull(),
32
+
state: text({enum: ['queued','discovered','discarded','scrobbled','failed','duped']}).notNull(),
33
33
// https://orm.drizzle.team/docs/indexes-constraints#foreign-key
34
34
parentId: integer().references((): AnySQLiteColumn => plays.id, {onDelete: 'set null', onUpdate: 'cascade'}),
35
+
platformId: text(),
35
36
compacted: text()
36
37
}, (table) => [
37
38
index("play_parent_id_idx").on(table.parentId),
···
39
40
uniqueIndex("play_uid_idx").on(table.uid),
40
41
index("play_playedAt_idx").on(table.playedAt),
41
42
index("play_seenAt_idx").on(table.seenAt),
43
+
index("play_platform_idx").on(table.platformId)
42
44
]);
43
45
44
46
export const playInputs = sqliteTable("play_inputs", {
+1
-1
src/backend/server/api.ts
+1
-1
src/backend/server/api.ts
+80
-25
src/backend/sources/AbstractSource.ts
+80
-25
src/backend/sources/AbstractSource.ts
···
2
2
import dayjs, { Dayjs } from "dayjs";
3
3
import { EventEmitter } from "events";
4
4
import { FixedSizeList } from "fixed-size-list";
5
-
import { PlayObject } from "../../core/Atomic.js";
5
+
import { JsonPlayObject, PlayObject } from "../../core/Atomic.js";
6
6
import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js";
7
7
import AbstractComponent from "../common/AbstractComponent.js";
8
8
import {
···
31
31
sleep,
32
32
sortByOldestPlayDate,
33
33
} from "../utils.js";
34
-
import { sortByNewestPlayDate } from '../../core/PlayUtils.js';
34
+
import { genGroupIdStr, sortByNewestPlayDate } from '../../core/PlayUtils.js';
35
35
import { formatNumber } from '../../core/DataUtils.js';
36
36
import { timeToHumanTimestamp } from "../../core/TimeUtils.js";
37
37
import { todayAwareFormat } from "../../core/TimeUtils.js";
···
46
46
import { normalizeStr } from '../utils/StringUtils.js';
47
47
import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x';
48
48
import { AbortedError, generateLoggableAbortReason } from '../common/errors/MSErrors.js';
49
+
import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts } from '../common/database/drizzle/repositories/PlayRepository.js';
50
+
import { asPlay } from '../../core/PlayMarshalUtils.js';
49
51
50
52
export interface RecentlyPlayedOptions {
51
53
limit?: number
···
105
107
106
108
declare protected componentType: 'source';
107
109
110
+
protected playRepo: DrizzlePlayRepository;
111
+
108
112
constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) {
109
113
super(config);
110
114
this.componentType = 'source';
···
122
126
this.emitter = emitter;
123
127
124
128
this.discoveredCounter = getRoot().items.sourceMetics.discovered;
129
+
this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger});
125
130
}
126
131
127
132
async [Symbol.asyncDispose]() {
···
133
138
protected async postCache(): Promise<void> {
134
139
await super.postCache();
135
140
this.generateStaggerMappers();
141
+
}
142
+
143
+
protected async postDatabase(): Promise<void> {
144
+
this.tracksDiscovered = this.dbComponent.countLive;
136
145
}
137
146
138
147
protected generateStaggerMappers() {
···
203
212
// TODO make this more descriptive? or move it elsewhere
204
213
recentlyPlayedTrackIsValid = (playObj: PlayObject) => true
205
214
206
-
protected addPlayToDiscovered = (play: PlayObject) => {
215
+
protected addPlayToDiscovered = async (play: PlayObject) => {
207
216
const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID;
208
-
const list = this.recentDiscoveredPlays.get(platformId) ?? new FixedSizeList<ProgressAwarePlayObject>(200);
209
-
list.add(play);
210
-
this.recentDiscoveredPlays.set(platformId, list);
217
+
const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, componentId: this.dbComponent.id, state: 'discovered'}))]);
218
+
const recentPlays = await this.getRecentlyDiscoveredPlaysByPlatform(platformId, false);
219
+
// only need to update if its already in memory,
220
+
// and better to update in-memory than clear cache so we aren't refetching from db on every discover
221
+
if(recentPlays !== undefined) {
222
+
recentPlays.push(play);
223
+
recentPlays.sort(sortByOldestPlayDate);
224
+
this.cache.cacheDb.set(this.recentDiscoveredCacheKey(platformId), recentPlays, '2m');
225
+
}
226
+
const platformIds = await this.getRecentPlatformIds(false);
227
+
if(platformIds !== undefined && !platformIds.includes(genGroupIdStr(platformId))) {
228
+
this.cache.cacheDb.set(this.recentPlatformsCacheKey(), platformIds, '10m');
229
+
}
211
230
this.tracksDiscovered++;
212
231
this.logger.info(`Discovered => ${buildTrackString(play)}`);
213
232
this.emitEvent('discovered', {play});
214
233
this.discoveredCounter.labels(this.getPrometheusLabels()).inc();
215
234
}
216
235
217
-
getFlatRecentlyDiscoveredPlays = (): PlayObject[] =>
218
-
Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate)
219
-
236
+
getFlatRecentlyDiscoveredPlays = async (): Promise<PlayObject[]> => {
237
+
const platforms = await this.getRecentPlatformIds();
238
+
const list: PlayObject[][] = [];
239
+
for(const platformId of platforms) {
240
+
list.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId));
241
+
}
242
+
return list.flat().sort(sortByNewestPlayDate);
243
+
//Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate)
244
+
}
245
+
246
+
protected recentDiscoveredCacheKey = (platformId: PlayPlatformId | string) => {
247
+
const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId);
248
+
return `recent-${this.dbComponent.id}-${platformStr}`;
249
+
}
250
+
protected recentPlatformsCacheKey = () => {
251
+
return `recentPlatformIds-${this.dbComponent.id}`;
252
+
}
253
+
254
+
getRecentlyDiscoveredPlaysByPlatform = async (platformId: PlayPlatformId | string, hydrate: boolean = true): Promise<PlayObject[]> => {
255
+
256
+
const platformStr = typeof platformId === 'string' ? platformId : genGroupIdStr(platformId);
257
+
const cacheKey = this.recentDiscoveredCacheKey(platformId);
220
258
221
-
getRecentlyDiscoveredPlaysByPlatform = (platformId: PlayPlatformId): PlayObject[] => {
222
-
const list = this.recentDiscoveredPlays.get(platformId);
223
-
if (list !== undefined) {
224
-
const data = [...list.data];
225
-
data.sort(sortByOldestPlayDate);
226
-
return data;
259
+
let list = await this.cache.cacheDb.get<PlayObject[]>(cacheKey);
260
+
if(list === undefined && hydrate) {
261
+
list = (await this.playRepo.findPlays({
262
+
platformId: platformStr,
263
+
componentId: this.dbComponent.id,
264
+
stateNot: ['queued'],
265
+
order: 'desc',
266
+
sort: 'playedAt',
267
+
limit: 200
268
+
})).map(x => asPlay(x.play))
269
+
list.sort(sortByOldestPlayDate);
270
+
await this.cache.cacheDb.set<PlayObject[]>(cacheKey, list, '2m');
271
+
}
272
+
return list;
273
+
}
274
+
275
+
protected getRecentPlatformIds = async (hydrate: boolean = true) => {
276
+
const cacheKey = this.recentPlatformsCacheKey();
277
+
let list = await this.cache.cacheDb.get<string[]>(cacheKey);
278
+
if(list === undefined && hydrate) {
279
+
list = await this.playRepo.selectRecentDistinctPlatforms(this.dbComponent.id);
280
+
await this.cache.cacheDb.set<string[]>(cacheKey, list, '10m');
227
281
}
228
-
return [];
282
+
return list;
229
283
}
230
284
231
-
protected getExistingDiscoveredLists = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject[][] => {
285
+
protected getExistingDiscoveredLists = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject[][]> => {
232
286
const lists: PlayObject[][] = [];
233
287
if(opts.checkAll !== true) {
234
-
lists.push(this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID));
288
+
lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID));
235
289
} else {
290
+
const platforms = await this.getRecentPlatformIds();
236
291
// get as many as we can, optionally filtering by user
237
-
this.recentDiscoveredPlays.forEach((list, platformId) => {
292
+
for(const platformId of platforms) {
238
293
if(play.meta.user !== undefined) {
239
294
if(platformId[1] === NO_USER || platformId[1] === play.meta.user) {
240
-
lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId));
295
+
lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId));
241
296
}
242
297
} else {
243
-
lists.push(this.getRecentlyDiscoveredPlaysByPlatform(platformId));
298
+
lists.push(await this.getRecentlyDiscoveredPlaysByPlatform(platformId));
244
299
}
245
-
});
300
+
}
246
301
}
247
302
return lists;
248
303
}
249
304
250
305
existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
251
-
const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
306
+
const lists: PlayObject[][] = await this.getExistingDiscoveredLists(play, opts);
252
307
const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
253
308
for(const list of lists) {
254
309
···
275
330
options.signal?.throwIfAborted();
276
331
if(!(await this.alreadyDiscovered(play, options))) {
277
332
options.signal?.throwIfAborted()
278
-
this.addPlayToDiscovered(play);
333
+
await this.addPlayToDiscovered(play);
279
334
newDiscoveredPlays.push(play);
280
335
}
281
336
}
···
693
748
}
694
749
695
750
protected async doBuildComponentLogger(): Promise<void> {
696
-
if(this.config.options.logToFile) {
751
+
if(this.config?.options?.logToFile) {
697
752
this.logger.debug('Enabling component logger...');
698
753
const root = getRoot();
699
754
const stream = root.get('loggerStream');
+1
-1
src/backend/sources/DeezerInternalSource.ts
+1
-1
src/backend/sources/DeezerInternalSource.ts
···
330
330
331
331
332
332
existingDiscovered = async (play: PlayObject, opts: {checkAll?: boolean} = {}): Promise<PlayObject | undefined> => {
333
-
const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts);
333
+
const lists: PlayObject[][] = await this.getExistingDiscoveredLists(play, opts);
334
334
const candidate = await this.transformPlay(play, TRANSFORM_HOOK.candidate);
335
335
for(const list of lists) {
336
336
const existing = await findAsync(list, async x => {
+1
-1
src/backend/sources/EndpointLastfmSource.ts
+1
-1
src/backend/sources/EndpointLastfmSource.ts
+1
-1
src/backend/sources/EndpointListenbrainzSource.ts
+1
-1
src/backend/sources/EndpointListenbrainzSource.ts
+1
-1
src/backend/sources/MemorySource.ts
+1
-1
src/backend/sources/MemorySource.ts
···
368
368
}
369
369
return [false, `${stPrefix} ${EXPECTED_NON_DISCOVERED_REASON}`]
370
370
} else {
371
-
const discoveredPlays = this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
371
+
const discoveredPlays = await this.getRecentlyDiscoveredPlaysByPlatform(genGroupId(candidate));
372
372
if (discoveredPlays.length === 0 || !playObjDataMatch(discoveredPlays[0], candidate)) {
373
373
// if most recent stateful play is not this track we'll add it
374
374
return [true,`${stPrefix} added after ${thresholdResultSummary(thresholdResults)}. Matched other recent play but could not determine time frame due to missing duration. Allowed due to not being last played track.`];
+1
-1
src/backend/sources/WebScrobblerSource.ts
+1
-1
src/backend/sources/WebScrobblerSource.ts
···
155
155
return baseFormatPlayObj(obj, play);
156
156
}
157
157
158
-
getRecentlyPlayed = async (options = {}) => this.getFlatRecentlyDiscoveredPlays()
158
+
getRecentlyPlayed = async (options = {}) => await this.getFlatRecentlyDiscoveredPlays()
159
159
160
160
isValidScrobble = (playObj: PlayObject) => {
161
161
if (playObj.meta?.scrobbleAllowed === false) {
+10
-3
src/backend/sources/YTMusicSource.ts
+10
-3
src/backend/sources/YTMusicSource.ts
···
23
23
import { todayAwareFormat } from "../../core/TimeUtils.js";
24
24
import { parseArrayFromMaybeString, parseArtistCredits, parseCredits } from "../utils/StringUtils.js";
25
25
import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js";
26
+
import { FixedSizeList } from "fixed-size-list";
26
27
27
28
export interface HistoryIngressResult {
28
29
plays: PlayObject[],
···
118
119
declare config: YTMusicSourceConfig
119
120
120
121
recentlyPlayed: PlayObject[] = [];
122
+
transientDiscovered: FixedSizeList<PlayObject> = new FixedSizeList<PlayObject>(200);
121
123
122
124
yti: Innertube;
123
125
userCode?: string;
···
149
151
this.config.options = {...rest, logDiff: diffVal};
150
152
}
151
153
}
154
+
155
+
this.emitter.on('discovered', (play) => {
156
+
this.transientDiscovered.add(play);
157
+
})
152
158
}
153
159
154
160
public additionalApiData(): Record<string, any> {
···
588
594
if(consistent && newPlays.length > 1) {
589
595
const interimPlays = newPlays.slice(0, newPlays.length - 1);
590
596
// check enough time has passed since last discovery
591
-
const discovered = this.getFlatRecentlyDiscoveredPlays();
597
+
const discovered = this.transientDiscovered.data;
592
598
if(discovered.length > 0) {
593
599
const lastDiscovered = discovered[0].data.playDate;
594
600
// the assumption in behavior is that user skips 1 or more tracks which then get recorded to YTM history
···
681
687
const reversedPlays = [...referencePlays];
682
688
// actual order they were discovered in (oldest to newest)
683
689
reversedPlays.reverse();
684
-
if(this.getFlatRecentlyDiscoveredPlays().length === 0) {
690
+
if(this.transientDiscovered.data.length === 0) {
685
691
// and add to discovered since its empty
686
692
for(const refPlay of reversedPlays) {
687
-
this.addPlayToDiscovered(refPlay);
693
+
//this.transientDiscovered.add(refPlay);
694
+
await this.addPlayToDiscovered(refPlay);
688
695
}
689
696
}
690
697
}
+31
-1
src/backend/tests/database/drizzle.test.ts
+31
-1
src/backend/tests/database/drizzle.test.ts
···
16
16
import { generateRandomObj } from '../../../core/tests/utils/fixtures.js';
17
17
import { generateArray } from '../../../core/DataUtils.js';
18
18
import { objectsEqual } from '../../utils/DataUtils.js';
19
-
import { eq } from 'drizzle-orm';
19
+
import { eq, sql } from 'drizzle-orm';
20
+
import { PlaySelect } from '../../common/database/drizzle/drizzleTypes.js';
20
21
21
22
// would be great to push migrations directly from schema but doesn't seem supported in newest beta
22
23
// https://github.com/drizzle-team/drizzle-orm/discussions/4373
···
528
529
expect(p2Plays).length(2);
529
530
expect(p2Plays[0]).to.eq(initialPlays[2].id);
530
531
expect(p2Plays[1]).to.eq(childPlays[0].id);
532
+
});
533
+
534
+
it('Get json property from play', async function () {
535
+
536
+
const db = getDb(':memory:', { workingDirectory: process.cwd() });
537
+
await migrateDb(db);
538
+
539
+
try {
540
+
541
+
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
542
+
543
+
const playRows = await db.insert(plays).values([
544
+
fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test1'}) }),
545
+
fixtureCreatePlay({ componentId: component[0].id, play: generatePlay({}, {source: 'test2'}) })
546
+
]).returning();
547
+
548
+
let result: PlaySelect[];
549
+
// https://github.com/drizzle-team/drizzle-orm/discussions/938#discussioncomment-6542336
550
+
result = await db.select().from(plays).where(
551
+
sql`json_extract(${plays.play}, '$.meta.source') = 'test1'`
552
+
);
553
+
554
+
expect(result).length(1);
555
+
expect(result[0].play.meta.source).eq('test1');
556
+
557
+
} catch (e) {
558
+
throw e;
559
+
}
560
+
db.$client.close();
531
561
});
532
562
533
563
});
+37
-31
src/backend/tests/source/source.test.ts
+37
-31
src/backend/tests/source/source.test.ts
···
24
24
25
25
26
26
const emitter = new EventEmitter();
27
-
const generateSource = () => {
28
-
return new TestSource('spotify', 'test', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
27
+
const generateSource = async () => {
28
+
const source = new TestSource('spotify', 'test-basic', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
29
+
await source.tryInitialize();
30
+
return source;
29
31
}
30
-
const generateMemorySource = (config: SourceConfig = {}) => {
31
-
const s = new TestMemorySource('spotify', 'test', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
32
-
s.buildTransformRules();
32
+
const generateMemorySource = async (config: SourceConfig = {}) => {
33
+
const s = new TestMemorySource('spotify', 'test-memory', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
34
+
await s.tryInitialize();
35
+
// s.buildTransformRules();
33
36
s.scheduler.stop();
34
37
return s;
35
38
}
36
39
37
-
const generateMemoryPositionalSource = (config: SourceConfig = {}) => {
38
-
const s = new TestMemoryPositionalSource('spotify', 'test', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
39
-
s.buildTransformRules();
40
+
const generateMemoryPositionalSource = async (config: SourceConfig = {}) => {
41
+
const s = new TestMemoryPositionalSource('spotify', 'test-positional', config, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
42
+
await s.tryInitialize();
43
+
//s.buildTransformRules();
40
44
s.scheduler.stop();
41
45
return s;
42
46
}
···
44
48
describe('Sources use transform plays correctly', function () {
45
49
46
50
it('Transforms play on preCompare', async function() {
47
-
await using source = generateSource();
51
+
await using source = await generateSource();
48
52
source.config.options = {
49
53
playTransform: {
50
54
preCompare: {
···
67
71
});
68
72
69
73
it('Transforms play on postCompare', async function() {
70
-
await using source = generateSource();
74
+
await using source = await generateSource();
71
75
source.config.options = {
72
76
playTransform: {
73
77
postCompare: {
···
96
100
});
97
101
98
102
it('Transforms play existing comparison', async function() {
99
-
await using source = generateSource();
103
+
await using source = await generateSource();
100
104
source.config.options = {
101
105
playTransform: {
102
106
compare: {
···
123
127
});
124
128
125
129
it('Transforms play candidate comparison', async function() {
126
-
await using source = generateSource();
130
+
await using source = await generateSource();
127
131
source.config.options = {
128
132
playTransform: {
129
133
compare: {
···
192
196
setRtTick(1);
193
197
});
194
198
195
-
const cleanedUpDuration = async (generateSource: (config: SourceConfig) => MemorySource) => {
196
-
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
199
+
const cleanedUpDuration = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
200
+
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
197
201
const initialDate = dayjs();
198
202
const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, stateUpdatedAt: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
199
203
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
···
235
239
await cleanedUpDuration(generateMemoryPositionalSource);
236
240
});
237
241
238
-
const noScrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => MemorySource) => {
242
+
const noScrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
239
243
240
-
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
244
+
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
241
245
const initialDate = dayjs();
242
246
const initialState = generatePlayerStateData({position: 0, playData: {duration: 50}, stateUpdatedAt: initialDate, status: REPORTED_PLAYER_STATUSES.playing});
243
247
expect((await source.processRecentPlays([initialState])).length).to.be.eq(0);
···
303
307
await noScrobbleRediscoveryOnActive(generateMemoryPositionalSource);
304
308
});
305
309
306
-
const noScrobbleStale = async (generateSource: (config: SourceConfig) => MemorySource) => {
310
+
const noScrobbleStale = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
307
311
308
-
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
312
+
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
309
313
const initialDate = dayjs();
310
314
311
315
// if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s
···
348
352
await noScrobbleStale(generateMemoryPositionalSource);
349
353
});
350
354
351
-
const scrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => MemorySource) => {
355
+
const scrobbleRediscoveryOnActive = async (generateSource: (config: SourceConfig) => Promise<MemorySource>) => {
352
356
353
-
await using source = generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
357
+
await using source = await generateSource({data: {staleAfter: 21, orphanedAfter: 40}, options: {}});
354
358
const initialDate = dayjs();
355
359
356
360
// if player incorrectly counted stale time then 30s of actual play + 20s of stale time > scrobble threshold of 50% of 90s
···
421
425
});
422
426
});
423
427
424
-
const generateDeezerSource = (options: DeezerInternalSourceOptions = {}) => {
425
-
return new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
428
+
const generateDeezerSource = async (options: DeezerInternalSourceOptions = {}) => {
429
+
const source = new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
430
+
await source.tryInitialize();
431
+
return source;
426
432
}
427
433
const firstPlayDate = dayjs().subtract(1, 'hour');
428
434
const normalizedPlays = normalizePlays(generatePlays(6), {initialDate: firstPlayDate});
···
438
444
const fuzzyPlay = clone(targetPlay);
439
445
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
440
446
441
-
const source = generateDeezerSource();
447
+
const source = await generateDeezerSource();
442
448
source.discover([...normalizedPlays, interimPlay]);
443
449
444
450
const discovered = await source.discover([fuzzyPlay]);
···
455
461
const fuzzyPlay = clone(targetPlay);
456
462
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
457
463
458
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
464
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
459
465
await source.discover([...normalizedPlays, interimPlay]);
460
466
461
467
const discovered = await source.discover([fuzzyPlay]);
···
468
474
const fuzzyPlay = clone(targetPlay);
469
475
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
470
476
471
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
477
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
472
478
await source.discover(normalizedPlays);
473
479
474
480
const discovered = await source.discover([fuzzyPlay]);
···
482
488
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
483
489
const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
484
490
485
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: true});
491
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: true});
486
492
const discovered = await source.discover(morePlays);
487
493
488
494
expect(discovered.length).to.eq(morePlays.length);
···
497
503
const fuzzyPlay = clone(targetPlay);
498
504
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
499
505
500
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
506
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
501
507
await source.discover([...normalizedPlays, interimPlay]);
502
508
503
509
const discovered = await source.discover([fuzzyPlay]);
···
511
517
const duringPlay = clone(targetPlay);
512
518
duringPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration * 0.5, 's');
513
519
514
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
520
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
515
521
await source.discover([...normalizedPlays, interimPlay]);
516
522
517
523
const discovered = await source.discover([duringPlay]);
···
525
531
const fuzzyPlay = clone(targetPlay);
526
532
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration + 39, 's');
527
533
528
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
534
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
529
535
await source.discover([...normalizedPlays, interimPlay]);
530
536
531
537
const discovered = await source.discover([fuzzyPlay]);
···
538
544
const fuzzyPlay = clone(targetPlay);
539
545
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
540
546
541
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
547
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
542
548
await source.discover(normalizedPlays);
543
549
544
550
const discovered = await source.discover([fuzzyPlay]);
···
552
558
fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's');
553
559
const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate});
554
560
555
-
await using source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
561
+
await using source = await generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'});
556
562
const discovered = await source.discover(morePlays);
557
563
558
564
expect(discovered.length).to.eq(morePlays.length - 1);
+8
-7
src/backend/tests/ytm/ytm.test.ts
+8
-7
src/backend/tests/ytm/ytm.test.ts
···
14
14
15
15
chai.use(asPromised);
16
16
17
-
const createYtSource = (opts?: {
17
+
const createYtSource = async (opts?: {
18
18
config?: YTMusicSourceConfig
19
19
emitter?: EventEmitter
20
20
}) => {
···
27
27
emitter = new EventEmitter
28
28
} = opts || {};
29
29
const source = new YTMusicSource('test', config, { localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test' }, emitter);
30
+
await source.buildDatabase();
30
31
source.buildTransformRules();
31
32
return source;
32
33
}
···
48
49
49
50
it(`Adds new, prepended track`, async function () {
50
51
51
-
const source = createYtSource();
52
+
const source = await createYtSource();
52
53
53
54
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
54
55
···
72
73
73
74
it(`Adds bumped, prepended track`, async function () {
74
75
75
-
const source = createYtSource();
76
+
const source = await createYtSource();
76
77
77
78
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
78
79
···
98
99
99
100
it(`Does not add appended track`, async function () {
100
101
101
-
const source = createYtSource();
102
+
const source = await createYtSource();
102
103
103
104
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
104
105
···
122
123
123
124
this.timeout(3700);
124
125
125
-
const source = createYtSource();
126
+
const source = await createYtSource();
126
127
127
128
const plays = [...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(10, 'minutes')}, { comment: 'Yesterday' })];
128
129
···
166
167
167
168
it(`Does not add skipped plays`, async function () {
168
169
169
-
const source = createYtSource();
170
+
const source = await createYtSource();
170
171
171
172
const plays = [...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(20, 'seconds')}, { comment: 'Yesterday' })];
172
173
···
193
194
194
195
it(`Adds interim plays when discover time is plausible`, async function () {
195
196
196
-
const source = createYtSource();
197
+
const source = await createYtSource();
197
198
198
199
const plays = [...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Today' }), ...generatePlays(10, {playDate: dayjs().subtract(2, 'minutes')}, { comment: 'Yesterday' })];
199
200
+1
-1
src/core/Atomic.ts
+1
-1
src/core/Atomic.ts
+1
-1
src/core/PlayMarshalUtils.ts
+1
-1
src/core/PlayMarshalUtils.ts
···
56
56
return cloned as unknown as JsonPlayObject;
57
57
};
58
58
59
-
export const asPlay = (data: JsonPlayObject): PlayObject => {
59
+
export const asPlay = (data: JsonPlayObject | PlayObject): PlayObject => {
60
60
const cloned = clone(data);
61
61
new Traverse(cloned).forEach((ctx, x) => {
62
62
if (shouldBlock(ctx)) {