feat(deezer internal): Add fuzzy discovery ignore option · foxxmd.dev/multi-scrobbler@6bd841f

+6 -1

src/backend/common/infrastructure/config/source/deezer.ts

··· 1 1 import { Second } from "../../../../../core/Atomic.js"; 2 2 import { PollingOptions } from "../common.js"; 3 - import { CommonSourceConfig, CommonSourceData } from "./index.js"; 3 + import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js"; 4 4 5 5 export interface DeezerData extends CommonSourceData, PollingOptions { 6 6 /** ··· 43 43 44 44 export interface DeezerInternalSourceConfig extends CommonSourceConfig { 45 45 data: DeezerInternalData 46 + options: DeezerInternalSourceOptions 47 + } 48 + 49 + export interface DeezerInternalSourceOptions extends CommonSourceOptions { 50 + fuzzyDiscoveryIgnore?: boolean | 'aggressive' 46 51 } 47 52 48 53 export interface DeezerInternalAIOConfig extends DeezerInternalSourceConfig {

+8 -4

src/backend/sources/AbstractSource.ts

··· 38 38 import { componentFileLogger } from '../common/logging.js'; 39 39 import { WebhookPayload } from '../common/infrastructure/config/health/webhooks.js'; 40 40 import { messageWithCauses, messageWithCausesTruncatedDefault } from '../utils/ErrorUtils.js'; 41 + import { genericSourcePlayMatch } from '../utils/PlayComparisonUtils.js'; 41 42 42 43 export interface RecentlyPlayedOptions { 43 44 limit?: number ··· 152 153 return []; 153 154 } 154 155 155 - existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => { 156 + protected getExistingDiscoveredLists = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject[][] => { 156 157 const lists: PlayObject[][] = []; 157 158 if(opts.checkAll !== true) { 158 159 lists.push(this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID)); ··· 168 169 } 169 170 }); 170 171 } 172 + return lists; 173 + } 174 + 175 + existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => { 176 + const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts); 171 177 const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate); 172 178 for(const list of lists) { 173 179 const existing = list.find(x => { 174 180 const e = this.transformPlay(x, TRANSFORM_HOOK.existing); 175 - return playObjDataMatch(e, candidate) && temporalAccuracyIsAtLeast(TA_CLOSE, comparePlayTemporally(e, candidate).match) 181 + return genericSourcePlayMatch(e, candidate, TA_CLOSE); 176 182 }); 177 183 if(existing) { 178 184 return existing; 179 185 } 180 186 } 181 187 return undefined; 182 - //const list = this.getRecentlyDiscoveredPlaysByPlatform(this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID); 183 - //return list.find(x => playObjDataMatch(x, play) && closePlayDate(x, play)); 184 188 } 185 189 186 190 alreadyDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): boolean => {

+41 -3

src/backend/sources/DeezerInternalSource.ts

··· 1 1 import dayjs from "dayjs"; 2 2 import EventEmitter from "events"; 3 3 import request, { Request, Response, SuperAgent } from 'superagent'; 4 - import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; 5 - import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; 4 + import { PlayObject, SOURCE_SOT, TA_CLOSE, TA_FUZZY } from "../../core/Atomic.js"; 5 + import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig, TRANSFORM_HOOK } from "../common/infrastructure/Atomic.js"; 6 6 import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; 7 - import { parseRetryAfterSecsFromObj, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; 7 + import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; 8 8 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; 9 9 import { CookieJar, Cookie } from 'tough-cookie'; 10 10 import { MixedCookieAgent } from 'http-cookie-agent/http'; 11 11 import MemorySource from "./MemorySource.js"; 12 + import { genericSourcePlayMatch } from "../utils/PlayComparisonUtils.js"; 12 13 13 14 interface DeezerHistoryResponse { 14 15 errors: [] ··· 195 196 } 196 197 197 198 protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options}) 199 + 200 + 201 + existingDiscovered = (play: PlayObject, opts: {checkAll?: boolean} = {}): PlayObject | undefined => { 202 + const lists: PlayObject[][] = this.getExistingDiscoveredLists(play, opts); 203 + const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate); 204 + for(const list of lists) { 205 + const existing = list.find(x => { 206 + const e = this.transformPlay(x, TRANSFORM_HOOK.existing); 207 + return genericSourcePlayMatch(e, candidate, TA_CLOSE); 208 + }); 209 + if(existing) { 210 + return existing; 211 + } 212 + if(this.config.options?.fuzzyDiscoveryIgnore === true || this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { 213 + const fuzzyIndex = list.findIndex(x => { 214 + const e = this.transformPlay(x, TRANSFORM_HOOK.existing); 215 + return genericSourcePlayMatch(e, candidate, TA_FUZZY); 216 + }); 217 + if(fuzzyIndex !== -1) { 218 + if(this.config.options?.fuzzyDiscoveryIgnore === 'aggressive') { 219 + // always return fuzzy match as existing 220 + // likely will make MS miss scrobbles for repeated plays 221 + return list[fuzzyIndex]; 222 + } 223 + if(fuzzyIndex + 1 === list.length || playObjDataMatch(list[fuzzyIndex], list[fuzzyIndex + 1])) { 224 + // last discovered play was this one, or next played play was also this one 225 + // so we'll assume this means the play is on repeat, don't count as existing 226 + return undefined; 227 + } 228 + // next played play was *not* this one (Deezer reports play between candidate TS and fuzzy match) 229 + // so this is likely a duplicate deezer should not have reported 230 + return list[fuzzyIndex]; 231 + } 232 + } 233 + } 234 + return undefined; 235 + } 198 236 } 199 237 200 238 const setRequestHeaders = (req: Request, userAgent: string = 'Mozilla/5.0 (X11; Linux i686; rv:135.0) Gecko/20100101 Firefox/135.0') => {

+115 -1

src/backend/tests/source/source.test.ts

··· 6 6 import pEvent from "p-event"; 7 7 import clone from 'clone'; 8 8 import { PlayObject } from "../../../core/Atomic.js"; 9 - import { generatePlay, generatePlayerStateData } from "../utils/PlayTestUtils.js"; 9 + import { generatePlay, generatePlayerStateData, generatePlays, normalizePlays } from "../utils/PlayTestUtils.js"; 10 10 import { TestMemoryPositionalSource, TestMemorySource, TestSource } from "./TestSource.js"; 11 11 import spotifyPayload from '../plays/spotifyCurrentPlaybackState.json'; 12 12 import SpotifySource from "../../sources/SpotifySource.js"; ··· 19 19 import { DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD } from "../../common/infrastructure/Atomic.js"; 20 20 import { RT_TICK_DEFAULT, setRtTick } from "../../sources/PlayerState/RealtimePlayer.js"; 21 21 import { sleep } from "../../utils.js"; 22 + import DeezerInternalSource from "../../sources/DeezerInternalSource.js"; 23 + import { DeezerInternalSourceOptions } from "../../common/infrastructure/config/source/deezer.js"; 22 24 23 25 chai.use(asPromised); 24 26 ··· 457 459 expect(results.duration.passes).is.true; 458 460 expect(results.passes).is.true; 459 461 }); 462 + }); 463 + 464 + const generateDeezerSource = (options: DeezerInternalSourceOptions = {}) => { 465 + return new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter); 466 + } 467 + const firstPlayDate = dayjs().subtract(1, 'hour'); 468 + const normalizedPlays = normalizePlays(generatePlays(6), {initialDate: firstPlayDate}); 469 + const lastPlay = normalizedPlays[normalizePlays.length - 1]; 470 + 471 + describe('Deezer Internal Source', function() { 472 + 473 + describe('When fuzzyDiscoveryIgnore is not defined or false', function () { 474 + 475 + it('discovers fuzzy play', function() { 476 + const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); 477 + const targetPlay = normalizedPlays[normalizedPlays.length - 2] 478 + const fuzzyPlay = clone(targetPlay); 479 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 480 + 481 + const source = generateDeezerSource(); 482 + source.discover([...normalizedPlays, interimPlay]); 483 + 484 + const discovered = source.discover([fuzzyPlay]); 485 + 486 + expect(discovered.length).to.eq(1); 487 + }); 488 + }); 489 + 490 + describe('When fuzzyDiscoveryIgnore is true', function () { 491 + 492 + it('does not discover fuzzy play with interim plays', function() { 493 + const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); 494 + const targetPlay = normalizedPlays[normalizedPlays.length - 2] 495 + const fuzzyPlay = clone(targetPlay); 496 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 497 + 498 + const source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); 499 + source.discover([...normalizedPlays, interimPlay]); 500 + 501 + const discovered = source.discover([fuzzyPlay]); 502 + 503 + expect(discovered.length).to.eq(0); 504 + }); 505 + 506 + it('discovers fuzzy play when it is the last play ', function() { 507 + const targetPlay = normalizedPlays[normalizedPlays.length - 1] 508 + const fuzzyPlay = clone(targetPlay); 509 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 510 + 511 + const source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); 512 + source.discover(normalizedPlays); 513 + 514 + const discovered = source.discover([fuzzyPlay]); 515 + 516 + expect(discovered.length).to.eq(1); 517 + }); 518 + 519 + it('discovers fuzzy play when it is played consecutively', function() { 520 + const targetPlay = normalizedPlays[normalizedPlays.length - 1] 521 + const fuzzyPlay = clone(targetPlay); 522 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 523 + const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); 524 + 525 + const source = generateDeezerSource({fuzzyDiscoveryIgnore: true}); 526 + const discovered = source.discover(morePlays); 527 + 528 + expect(discovered.length).to.eq(morePlays.length); 529 + }); 530 + }); 531 + 532 + describe('When fuzzyDiscoveryIgnore is aggressive', function () { 533 + 534 + it('does not discover fuzzy play with interim plays', function() { 535 + const interimPlay = generatePlay({playDate: lastPlay.data.playDate.add(15, 's'), duration: 80}); 536 + const targetPlay = normalizedPlays[normalizedPlays.length - 2] 537 + const fuzzyPlay = clone(targetPlay); 538 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 539 + 540 + const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); 541 + source.discover([...normalizedPlays, interimPlay]); 542 + 543 + const discovered = source.discover([fuzzyPlay]); 544 + 545 + expect(discovered.length).to.eq(0); 546 + }); 547 + 548 + it('it does not discover fuzzy play when it is the last play ', function() { 549 + const targetPlay = normalizedPlays[normalizedPlays.length - 1] 550 + const fuzzyPlay = clone(targetPlay); 551 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 552 + 553 + const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); 554 + source.discover(normalizedPlays); 555 + 556 + const discovered = source.discover([fuzzyPlay]); 557 + 558 + expect(discovered.length).to.eq(0); 559 + }); 560 + 561 + it('does not discover fuzzy play when it is played consecutively', function() { 562 + const targetPlay = normalizedPlays[normalizedPlays.length - 1] 563 + const fuzzyPlay = clone(targetPlay); 564 + fuzzyPlay.data.playDate = targetPlay.data.playDate.add(targetPlay.data.duration, 's'); 565 + const morePlays = normalizePlays([...normalizedPlays, fuzzyPlay, ...generatePlays(2)], {initialDate: firstPlayDate}); 566 + 567 + const source = generateDeezerSource({fuzzyDiscoveryIgnore: 'aggressive'}); 568 + const discovered = source.discover(morePlays); 569 + 570 + expect(discovered.length).to.eq(morePlays.length - 1); 571 + }); 572 + 573 + }); 460 574 });

+35 -2

src/backend/tests/utils/playComparisons.test.ts

··· 1 1 import { loggerTest } from "@foxxmd/logging"; 2 - import { assert } from 'chai'; 2 + import { assert, expect } from 'chai'; 3 3 import clone from "clone"; 4 4 import { describe, it } from 'mocha'; 5 - import { playsAreAddedOnly, playsAreBumpedOnly, playsAreSortConsistent } from "../../utils/PlayComparisonUtils.js"; 5 + import { genericSourcePlayMatch, playsAreAddedOnly, playsAreBumpedOnly, playsAreSortConsistent } from "../../utils/PlayComparisonUtils.js"; 6 6 import { generatePlay, generatePlays } from "./PlayTestUtils.js"; 7 7 import { PlayObject } from "../../../core/Atomic.js"; 8 8 ··· 248 248 assert.equal(addType, 'append'); 249 249 }); 250 250 251 + }); 252 + 253 + describe('Source Play Comparisons', function() { 254 + 255 + describe('Generic Source Comparison', function() { 256 + 257 + it('matches identical plays', function() { 258 + expect(genericSourcePlayMatch(newPlay, newPlay)).to.be.true; 259 + }); 260 + 261 + it('matches identical plays with close timestamps', function() { 262 + const closePlay = clone(newPlay); 263 + closePlay.data.playDate = closePlay.data.playDate.add(3, 's'); 264 + expect(genericSourcePlayMatch(newPlay, closePlay)).to.be.true; 265 + }); 266 + 267 + it('does not match unique plays', function() { 268 + expect(genericSourcePlayMatch(newPlay, generatePlay())).to.be.false; 269 + }); 270 + 271 + it('does not match unique plays with exact timestamps', function() { 272 + const diffPlay = generatePlay(); 273 + diffPlay.data.playDate = newPlay.data.playDate; 274 + expect(genericSourcePlayMatch(newPlay, diffPlay)).to.be.false; 275 + }); 276 + 277 + it('does not match unique plays with close timestamps', function() { 278 + const diffPlay = generatePlay(); 279 + diffPlay.data.playDate = newPlay.data.playDate.add(3, 's'); 280 + expect(genericSourcePlayMatch(newPlay, diffPlay)).to.be.false; 281 + }); 282 + }); 283 + 251 284 }); 252 285 });

+5 -1

src/backend/utils/PlayComparisonUtils.ts

··· 1 1 import { getListDiff, ListDiff } from "@donedeal0/superdiff"; 2 - import { PlayObject } from "../../core/Atomic.js"; 2 + import { PlayObject, TA_CLOSE, TemporalAccuracy } from "../../core/Atomic.js"; 3 3 import { buildTrackString } from "../../core/StringUtils.js"; 4 + import { playObjDataMatch } from "../utils.js"; 5 + import { comparePlayTemporally, temporalAccuracyIsAtLeast } from "./TimeUtils.js"; 4 6 5 7 6 8 export const metaInvariantTransform = (play: PlayObject): PlayObject => { ··· 221 223 return `${a} => ${b}`; 222 224 }).join('\n'); 223 225 } 226 + 227 + export const genericSourcePlayMatch = (a: PlayObject, b: PlayObject, t: TemporalAccuracy = TA_CLOSE): boolean => playObjDataMatch(a, b) && temporalAccuracyIsAtLeast(t, comparePlayTemporally(a, b).match);

+7

src/core/Atomic.ts

··· 246 246 247 247 export type TemporalAccuracy = 1 | 2 | 3 | false; 248 248 249 + /** Timestamp diffs are close to exact (less than or equal to 1 second difference) */ 249 250 export const TA_EXACT: TemporalAccuracy = 1; 251 + /** Timestamp diffs are within source reporting granularity margin-of-error (see lowGranularitySources): 252 + * normal granularity is 10 seconds 253 + * low granularity (subsonic usually) is 60 seconds 254 + */ 250 255 export const TA_CLOSE: TemporalAccuracy = 2; 256 + /** Timestamp diffs are not EXACT/CLOSE but Scrobble A's timestamp +/- duration is within 10 seconds of Scrobble B's timestamp */ 251 257 export const TA_FUZZY: TemporalAccuracy = 3; 258 + /** No correlation between timestamps */ 252 259 export const TA_NONE: TemporalAccuracy = false; 253 260 254 261 export interface TemporalPlayComparison {

Configure Feed

Configure Feed