foxxmd.dev / multi-scrobbler
[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients docs.multi-scrobbler.app
deezer docker jellyfin koito lastfm listenbrainz maloja mopidy mpris music music-assistant plex scrobble self-hosted spotify subsonic tautulli youtube-music
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

feat: Add flow control to transforms and fix bugs

FoxxMD (Nov 28, 2025, 11:18 PM UTC) a7df5a8d 883050e0

+271 -235
+59 -2
src/backend/common/AbstractComponent.ts
··· 43 43 } 44 44 45 45 public buildTransformRules() { 46 + this.logger.debug('Building transformer rules...'); 46 47 try { 47 48 this.doBuildTransformRules(); 48 49 } catch (e) { ··· 65 66 } = this.config; 66 67 67 68 if (playTransform === undefined) { 69 + this.logger.debug(`No rules found under property 'playTransform'`); 68 70 return; 69 71 } 70 72 ··· 82 84 existing, 83 85 postCompare; 84 86 87 + const builtHooks: string[] = []; 88 + const emptyHooks: string[] = []; 85 89 try { 86 90 preCompare = this.transformPartToStrong(preConfig); 91 + if(preCompare === undefined) { 92 + emptyHooks.push('preCompare') 93 + } else { 94 + builtHooks.push('preCompare'); 95 + } 87 96 } catch (e) { 88 97 throw new Error('preCompare was not valid', {cause: e}); 89 98 } 90 99 91 100 try { 92 101 candidate = this.transformPartToStrong(candidateConfig); 102 + if(candidate === undefined) { 103 + emptyHooks.push('candidate') 104 + } else { 105 + builtHooks.push('candidate'); 106 + } 93 107 } catch (e) { 94 108 throw new Error('candidate was not valid', {cause: e}); 95 109 } 96 110 97 111 try { 98 112 existing = this.transformPartToStrong(existingConfig); 113 + if(existing === undefined) { 114 + emptyHooks.push('existing') 115 + } else { 116 + builtHooks.push('existing'); 117 + } 99 118 } catch (e) { 100 119 throw new Error('existing was not valid', {cause: e}); 101 120 } 102 121 103 122 try { 104 123 postCompare = this.transformPartToStrong(postConfig); 124 + if(postCompare === undefined) { 125 + emptyHooks.push('postCompare') 126 + } else { 127 + builtHooks.push('postCompare'); 128 + } 105 129 } catch (e) { 106 130 throw new Error('postCompare was not valid', {cause: e}); 107 131 } 132 + 133 + this.logger.debug(`Hooks built. Configured: ${builtHooks.join(', ')} | Empty: ${emptyHooks.join(', ')}`); 108 134 109 135 this.transformRules = { 110 136 preCompare, ··· 155 181 } 156 182 157 183 let transformedPlay: PlayObject = play; 158 - const transformDetails: string[] = []; 184 + let transformDetails: string[] = []; 159 185 for(const hookItem of hook) { 160 186 161 - const newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay); 187 + const { 188 + onSuccess = 'continue', 189 + onFailure = 'stop', 190 + failureReturnPartial = false 191 + } = hookItem; 192 + 193 + let newTransformedPlay: PlayObject; 194 + let err: Error; 195 + try { 196 + newTransformedPlay = await this.transformManager.handleStage(hookItem, transformedPlay); 197 + } catch (e) { 198 + err = e; 199 + } 200 + 201 + if(err !== undefined) { 202 + if(onFailure === 'continue') { 203 + this.logger.warn(new Error('A transform encountered an error but continuing due to onFailure: continue', {cause: err})); 204 + } else { 205 + this.logger.error(new Error('Transform encountered an error', {cause: err})); 206 + if(!failureReturnPartial) { 207 + // rewind to original play so we don't return partial transform 208 + transformedPlay = play; 209 + transformDetails = []; 210 + } 211 + break; 212 + } 213 + } 162 214 163 215 if(!deepEqual(newTransformedPlay, transformedPlay)) { 164 216 transformDetails.push(`${hookItem.type} - ${buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})}`); 165 217 } 166 218 transformedPlay = newTransformedPlay; 219 + 220 + if(err === undefined && onSuccess === 'stop') { 221 + this.logger.debug('Stopping transform due to onSuccess: stop'); 222 + break; 223 + } 167 224 } 168 225 169 226 if(transformDetails.length > 0) {
+12 -4
src/backend/common/Cache.ts
··· 49 49 cacheScrobble: Cacheable; 50 50 cacheAuth: Cacheable; 51 51 regexCache: ReturnType<typeof cacheFunctions>; 52 + cacheTransform: Cacheable; 52 53 53 54 logger: Logger; 54 55 ··· 94 95 }; 95 96 96 97 this.regexCache = cacheFunctions(this.config.regex); 98 + this.cacheTransform = new Cacheable({primary: initMemoryCache({lruSize: 500})}); 97 99 } 98 100 99 101 init = async () => { ··· 101 103 //await this.initMetadataCache(); 102 104 await this.initScrobbleCache(); 103 105 await this.initAuthCache(); 106 + //this.cacheTransform = await this.initCacheable({provider: false, memory: {lruSize: 500}}, 'transform'); 104 107 } 105 108 106 109 protected initCacheable = async (config: CacheConfig, cacheFor: string) => { ··· 115 118 const ns = `ms-${cacheFor.toLocaleLowerCase()}`; 116 119 117 120 const cacheOpts: CacheableOptions = { 118 - primary: initMemoryCache({ namespace: ns }) 121 + primary: initMemoryCache({ namespace: ns, lruSize: config.memory?.lruSize, ttl: config.memory?.ttl }) 119 122 } 120 123 121 124 let secondaryCache: Keyv | KeyvStoreAdapter | undefined; ··· 180 183 181 184 182 185 export const initMemoryCache = (opts: Parameters<typeof createKeyv>[0] = {}): Keyv | KeyvStoreAdapter => { 186 + const { 187 + ttl = '1h', 188 + lruSize = 200, 189 + ...restOpts 190 + } = opts; 183 191 const memory = createKeyv({ 184 - ttl: '1h', 185 - lruSize: 200, 186 - ...opts, 192 + ttl, 193 + lruSize, 194 + ...restOpts, 187 195 useClone: false, 188 196 }); 189 197 // structuredClone does not work well with dayjs https://github.com/iamkun/dayjs/issues/2236
+20 -10
src/backend/common/infrastructure/Transform.ts
··· 1 1 import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; 2 2 3 - export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp { 4 - when?: WhenConditionsConfig 3 + export interface ConditionalSearchAndReplaceRegExp extends SearchAndReplaceRegExp, Whennable { 5 4 } 6 5 7 6 export type ConditionalSearchAndReplaceTerm = Omit<ConditionalSearchAndReplaceRegExp, 'test'> 8 7 export type SearchAndReplaceTerm = string | ConditionalSearchAndReplaceTerm; 9 8 export type ExternalMetadataTerm = boolean | undefined | Whennable; 10 9 11 - export type PlayTransformParts<T, Y = MaybeStageTyped> = Extract<PlayTransformStage<T>, Y> & { when?: WhenConditionsConfig }; 12 - export type PlayTransformUserParts<T> = PlayTransformUserStage<T[]> & { when?: WhenConditionsConfig }; 13 - export type PlayTransformMetaParts<T = ExternalMetadataTerm> = PlayTransformMetadataStage<T> & { when?: WhenConditionsConfig }; 10 + export type PlayTransformParts<T, Y = MaybeStageTyped> = Extract<PlayTransformStage<T>, Y> & Whennable; 11 + //export type PlayTransformUserParts<T> = PlayTransformUserStage<T[]> & { when?: WhenConditionsConfig }; 12 + //export type PlayTransformMetaParts<T = ExternalMetadataTerm> = PlayTransformMetadataStage<T> & { when?: WhenConditionsConfig }; 14 13 export type PlayTransformPartsArray<T, Y = MaybeStageTyped> = PlayTransformParts<T, Y>[]; 15 14 16 15 /** Represents the weakly-defined user config. May be an array of parts or one parts object */ ··· 47 46 when?: WhenConditionsConfig 48 47 } 49 48 50 - export interface StageConfig extends StageTypedConfig, Whennable {} 49 + export type FlowControlTerm = 'continue' | 'stop' 50 + 51 + export interface FlowControl { 52 + onSuccess: FlowControlTerm 53 + onFailure: FlowControlTerm 54 + failureReturnPartial: boolean 55 + } 56 + 57 + export interface StageConfig extends StageTypedConfig, Whennable, Partial<FlowControl> {} 51 58 52 59 export interface AtomicStageConfig<T> extends StageConfig, PlayTransformPartsAtomic<T> {} 53 60 ··· 55 62 type: StageType 56 63 } 57 64 58 - export interface PlayTransformMetadataStage<T = ExternalMetadataTerm> extends PlayTransformStageTyped<T> { 65 + export interface PlayTransformMetadataStage extends StageConfig, PlayTransformPartsAtomic<ExternalMetadataTerm> { 59 66 score?: number 60 67 // all?: ExternalMetadataTerm 61 68 type: StageTypeMetadata 62 69 } 63 70 64 - export interface PlayTransformUserStage<T> extends StageTypedConfig, PlayTransformPartsAtomic<T> { 71 + export interface PlayTransformUserStage<T> extends StageConfig, PlayTransformPartsAtomic<T> { 65 72 type: StageTypeUser 66 73 } 67 74 68 - export interface PlayTransformNativeStage<T> extends StageTypedConfig, PlayTransformPartsAtomic<T> { 75 + export interface PlayTransformNativeStage extends StageConfig, PlayTransformPartsAtomic<ExternalMetadataTerm> { 69 76 type: 'native' 70 77 } 71 78 79 + export interface PlayTransformGenericStage<T> extends StageConfig, PlayTransformPartsAtomic<T> { 80 + type: string 81 + } 72 82 73 83 export type UntypedPlayTransformUserStage<T> = Omit<PlayTransformUserStage<T>, 'type'> & {type?: never}; 74 84 75 - export type PlayTransformStage<T> = PlayTransformMetadataStage<T> | PlayTransformUserStage<T> | UntypedPlayTransformUserStage<T>; 85 + export type PlayTransformStage<T> = PlayTransformMetadataStage | PlayTransformUserStage<T> | PlayTransformNativeStage | UntypedPlayTransformUserStage<T> | PlayTransformGenericStage<any>; 76 86 77 87 /** Represents the plain json user-configured structure (input) */ 78 88 export interface PlayTransformHooksConfig<T> {
+31 -16
src/backend/common/transforms/AbstractTransformer.ts
··· 4 4 import { isStageTyped, testWhenConditions } from "../../utils/PlayTransformUtils.js"; 5 5 import AbstractInitializable from "../AbstractInitializable.js"; 6 6 import { StageConfig } from "../infrastructure/Transform.js"; 7 - import { cacheFunctions } from "@foxxmd/regex-buddy-core"; 7 + import { cacheFunctions, parseToRegexOrLiteralSearch, testMaybeRegex, searchAndReplace} from "@foxxmd/regex-buddy-core"; 8 + import { Cacheable } from "cacheable"; 9 + import { hashObject } from "../../utils/StringUtils.js"; 10 + import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js"; 8 11 9 12 export interface TransformerOptions { 10 13 logger: Logger 11 - regexCache: ReturnType<typeof cacheFunctions> 14 + regexCache?: ReturnType<typeof cacheFunctions> 15 + cache: Cacheable 16 + } 17 + 18 + export interface RegexObject { 19 + parseToRegexOrLiteralSearch: typeof parseToRegexOrLiteralSearch 20 + testMaybeRegex: typeof testMaybeRegex, 21 + searchAndReplace: typeof searchAndReplace 12 22 } 13 23 14 24 export default abstract class AbstractTransformer<T = any> extends AbstractInitializable { 15 25 16 26 declare config: TransformerCommonConfig; 27 + configHash: string; 17 28 18 29 transformType: string 19 30 20 - regexCache: ReturnType<typeof cacheFunctions> 31 + regex: RegexObject 32 + cache: Cacheable; 21 33 22 34 public constructor(config: TransformerCommon, options: TransformerOptions) { 23 35 super(config); 24 36 this.logger = childLogger(options.logger, ['Transformer', this.config.type, this.config.name]); 25 37 this.transformType = config.type; 26 - this.regexCache = options.regexCache; 38 + this.regex = options.regexCache ?? { searchAndReplace, testMaybeRegex, parseToRegexOrLiteralSearch }; 39 + this.cache = options.cache; 40 + this.configHash = hashObject(this.config); 27 41 } 28 42 29 43 public parseConfig(data: any) { ··· 37 51 38 52 public async handle(data: StageConfig, play: PlayObject): Promise<PlayObject> { 39 53 54 + const cacheKey = `${this.configHash}-${hashObject(data)}-${hashObject(playContentInvariantTransform(play))}` 55 + const cachedTransform = await this.cache.get<PlayObject>(cacheKey); 56 + if(cachedTransform !== undefined) { 57 + this.logger.debug('Cache hit'); 58 + return cachedTransform; 59 + } 60 + 40 61 if (data.when !== undefined) { 41 - if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { 62 + if (!testWhenConditions(data.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { 42 63 this.logger.debug('When condition not met, returning original Play'); 64 + await this.cache.set(cacheKey, play, '15s'); 43 65 return play; 44 66 } 45 67 } 46 68 47 - const { 48 - failOnFetch = false, 49 - throwOnFailure = false, 50 - } = this.config.options || {}; 51 - 52 69 let transformData: T; 53 70 try { 54 71 transformData = await this.getTransformerData(play); 55 72 } catch (e) { 56 - if (failOnFetch) { 57 - throw new Error(`Could not fetch transformer data`, { cause: e }); 58 - } 59 - this.logger.warn(new Error(`Could not fetch transformer data, returning original Play`, { cause: e })); 60 - return play; 73 + throw new Error(`Could not fetch transformer data`, { cause: e }); 61 74 } 62 75 63 76 try { ··· 67 80 return play; 68 81 } 69 82 70 - return await this.doHandle(data, play, transformData); 83 + const transformed = await this.doHandle(data, play, transformData); 84 + await this.cache.set(cacheKey, transformed, '15s'); 85 + return transformed; 71 86 } 72 87 73 88 protected abstract doHandle(data: StageConfig, play: PlayObject, transformData: T): Promise<PlayObject>;
+11 -10
src/backend/common/transforms/NativeTransformer.ts
··· 24 24 ignoreArtistsRegex: RegExp[] = []; 25 25 delimiters?: string[] 26 26 27 - // public constructor(config: NativeTransformerConfig, options: TransformerOptions) { 28 - // super(config, options); 29 - // } 27 + public constructor(config: NativeTransformerConfig, options: TransformerOptions) { 28 + super(config, options); 29 + } 30 30 31 31 protected async doBuildInitData(): Promise<true | string | undefined> { 32 32 if(this.config.data === undefined) { ··· 69 69 throw new Error(`NativeTransformer is only usable with 'native' type stages`); 70 70 } 71 71 72 - const stage: PlayTransformNativeStage<ExternalMetadataTerm> = { 72 + const stage: PlayTransformNativeStage = { 73 73 ...data, 74 74 type: 'native' 75 75 } 76 76 77 77 for (const k of ['artists', 'title', 'album']) { 78 - if (!(k in data)) { 78 + if (!(k in stage)) { 79 + stage[k] = true; 79 80 continue; 80 81 } 81 - if (Array.isArray(data[k])) { 82 + if (Array.isArray(stage[k])) { 82 83 throw new Error(`${k} must be a boolean or when object`); 83 84 } 84 - if (typeof data[k] === 'boolean') { 85 + if (typeof stage[k] === 'boolean') { 85 86 continue; 86 87 } 87 - if (typeof data[k] === 'object' && !isWhenCondition(data[k])) { 88 + if (typeof stage[k] === 'object' && !isWhenCondition(stage[k])) { 88 89 throw new Error(`${k} is not a valid when object`); 89 90 } 90 91 } ··· 132 133 return play.data.track; 133 134 } 134 135 protected async handleArtists(play: PlayObject, parts: ExternalMetadataTerm, transformData: PlayObject): Promise<string[] | undefined> { 135 - if (parts === false || parts === undefined) { 136 + if (parts === false) { 136 137 return play.data.artists; 137 138 } 138 139 if (typeof parts === 'object') { 139 140 if (parts.when !== undefined) { 140 - if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex })) { 141 + if (!testWhenConditions(parts.when, play, { testMaybeRegex: this.regex.testMaybeRegex })) { 141 142 this.logger.debug('When condition for artists not met, returning original artists'); 142 143 return play.data.artists; 143 144 }
+17 -9
src/backend/common/transforms/TransformerManager.ts
··· 10 10 11 11 export default class TransformerManager { 12 12 13 - logger: Logger; 14 - parentLogger: Logger; 15 - transformers: Map<string, AbstractTransformer[]> = new Map(); 16 - cache: MSCache; 13 + protected logger: Logger; 14 + protected parentLogger: Logger; 15 + protected transformers: Map<string, AbstractTransformer[]> = new Map(); 16 + protected cache: MSCache; 17 17 18 18 public constructor(logger: Logger, cache: MSCache) { 19 19 this.logger = childLogger(logger, 'Transformer Manager'); ··· 30 30 } 31 31 32 32 if (config.name !== undefined && transformers.some(x => x.config.name === config.name)) { 33 - throw new Error(`Cannot register ${config.type} with name '${config.name}' because an existing transformer already has that name`); 33 + throw new Error(`Cannot register ${config.type} transformer with name '${config.name}' because an existing transformer already has that name`); 34 34 } 35 35 const tName = config.name ?? `unnamed-${transformers.length + 1}`; 36 36 37 + this.logger.verbose(`Registering ${config.type} transformer with name '${tName}'`); 38 + 37 39 let t: AbstractTransformer; 38 40 switch (config.type) { 39 41 case 'user': 40 - t = new UserTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); 42 + t = new UserTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); 41 43 break; 42 44 case 'native': 43 - t = new NativeTransformer({ name: tName, logger: this.parentLogger, regexCache: this.cache.regexCache, ...config }); 45 + t = new NativeTransformer({ name: tName, ...config }, {logger: this.parentLogger, regexCache: this.cache.regexCache, cache: this.cache.cacheTransform}); 44 46 break; 45 47 default: 46 48 throw new Error(`No transformer of type '${config.type}' exists.`); 47 49 } 48 - 49 50 this.transformers.set(config.type, [...transformers, t]); 51 + this.logger.verbose(`${config.type} transformer with name '${tName}' registered`); 50 52 } 51 53 52 54 public async initTransformers() { 55 + this.logger.verbose('Initializing transformers...'); 53 56 for (const list of this.transformers.values()) { 54 57 for (const transformer of list) { 55 58 if (!transformer.isReady()) { ··· 64 67 } 65 68 } 66 69 } 70 + this.logger.verbose('Done initializing transformers'); 71 + } 72 + 73 + public hasTransformerType(type: string): boolean { 74 + return this.transformers.has(type); 67 75 } 68 76 69 77 protected getTransformerByStage(data: StageConfig): AbstractTransformer { ··· 72 80 throw new Error(`No transformer of type '${data.type}' is registered.`); 73 81 } 74 82 75 - if (list.length > 0 && (data as any).name === undefined) { 83 + if (list.length > 1 && (data as any).name === undefined) { 76 84 this.logger.warn(`More than one '${data.type}' transformer but name was not specified, using first registered`); 77 85 return list[0]; 78 86 } else {
+1 -1
src/backend/common/transforms/UserTransformer.ts
··· 39 39 } 40 40 41 41 protected generateMapper(play: PlayObject) { 42 - return (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({ ...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, { testMaybeRegex: this.regexCache.testMaybeRegex }) : undefined) }); 42 + return (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({ ...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, { testMaybeRegex: this.regex.testMaybeRegex }) : undefined) }); 43 43 } 44 44 45 45 protected async handleTitle(play: PlayObject, parts: ConditionalSearchAndReplaceRegExp[], _transformData: undefined): Promise<string | undefined> {
+12 -5
src/backend/ioc.ts
··· 29 29 loggerStream?: PassThrough 30 30 loggingConfig?: LogOptions 31 31 cache?: CacheConfigOptions | MSCache | (() => MSCache) 32 - transformerConfigs?: TransformerCommonConfig[] 32 + transformers?: TransformerCommonConfig[] 33 33 } 34 34 35 35 const createRoot = (options: RootOptions = {logger: loggerDebug}) => { ··· 41 41 loggingConfig, 42 42 logger, 43 43 cache, 44 - transformerConfigs = [], 44 + transformers = [], 45 45 } = options || {}; 46 46 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); 47 47 let disableWeb = dw; ··· 70 70 }); 71 71 72 72 const transformerManager = new TransformerManager(logger, maybeSingletonCache !== undefined ? maybeSingletonCache : cacheFunc()); 73 - transformerManager.register({type: 'user'}); 74 - transformerManager.register({type: 'native'}); 75 - for(const c of transformerConfigs) { 73 + for(const c of transformers) { 76 74 try { 77 75 transformerManager.register(c); 78 76 } catch (e) { 79 77 logger.warn(new Error('Could not register a transformer', {cause: e})); 80 78 } 79 + } 80 + if(transformers.length === 0) { 81 + logger.debug('No user-supplied transformer configs were found.'); 82 + } 83 + if(!transformerManager.hasTransformerType('user')) { 84 + transformerManager.register({type: 'user', name: 'MSDefault'}); 85 + } 86 + if(!transformerManager.hasTransformerType('native')) { 87 + transformerManager.register({type: 'native', name: 'MSDefault'}); 81 88 } 82 89 83 90 const portVal: number | string = process.env.PORT ?? port;
+88 -5
src/backend/tests/component/component.test.ts src/backend/tests/component/transformers.test.ts
··· 7 7 import { ConditionalSearchAndReplaceRegExp, STAGE_TYPES, STAGE_TYPES_METADATA, STAGE_TYPES_USER, TRANSFORM_HOOK } from "../../common/infrastructure/Transform.js"; 8 8 9 9 import { isConditionalSearchAndReplace } from "../../utils/PlayTransformUtils.js"; 10 - import { asPlays, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; 10 + import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; 11 11 import { WebhookPayload } from "../../common/infrastructure/config/health/webhooks.js"; 12 12 import { findCauseByMessage } from "../../utils/ErrorUtils.js"; 13 - import { ComAtprotoServerDescribeServer } from "@atproto/api"; 13 + import NativeTransformer from "../../common/transforms/NativeTransformer.js"; 14 + import { initMemoryCache } from "../../common/Cache.js"; 15 + import { Cacheable } from "cacheable"; 14 16 15 17 chai.use(asPromised); 16 18 ··· 29 31 const component = new TestComponent(); 30 32 component.logger = childLogger(loggerTest, 'App'); 31 33 34 + const memorycache = () => new Cacheable({primary: initMemoryCache()}) 35 + 32 36 describe('Play Transforms', function () { 33 37 34 38 beforeEach(function () { ··· 69 73 options: { 70 74 playTransform: { 71 75 preCompare: { 72 - // @ts-expect-error 73 76 type: "test", 74 77 title: ['something'] 75 78 } ··· 231 234 options: { 232 235 playTransform: { 233 236 preCompare: { 234 - // @ts-expect-error 235 237 type: t, 236 238 title: true 237 239 } ··· 400 402 }); 401 403 }); 402 404 405 + describe('Native Transformer', function () { 406 + 407 + it('Uses artist parsing functions', async function() { 408 + 409 + const t = new NativeTransformer({name: 'test', type: 'native'}, {logger: loggerTest, cache: memorycache()}); 410 + await t.tryInitialize(); 411 + 412 + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); 413 + const play = generatePlay({artists: [str]}); 414 + 415 + const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); 416 + expect(transformedPlay.data.artists).eql(primaries.concat(secondaries)); 417 + }); 418 + 419 + it('Ignores artists', async function() { 420 + 421 + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); 422 + 423 + const t = new NativeTransformer({name: 'test', type: 'native', data: {ignoreArtists: [str]}}, {logger: loggerTest, cache: memorycache()}); 424 + 425 + await t.tryInitialize(); 426 + 427 + const play = generatePlay({artists: [str], track: 'My Test'}); 428 + 429 + const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); 430 + expect(transformedPlay.data.artists).eql([str]); 431 + }); 432 + 433 + it('Uses custom delimiters artists', async function() { 434 + 435 + const [str, primaries, secondaries] = generateArtistsStr({primary: { 436 + max: 3, 437 + joiner: '•', 438 + spacedJoiners: true, 439 + ambiguousJoinedNames: false, 440 + trailingAmpersand: false, 441 + finalJoiner: false 442 + }}); 443 + 444 + const t = new NativeTransformer({name: 'test', type: 'native', data: {extraDelimiters: ['•']}}, {logger: loggerTest, cache: memorycache()}); 445 + 446 + await t.tryInitialize(); 447 + 448 + const play = generatePlay({artists: [str], track: 'My Test'}); 449 + 450 + const transformedPlay = await t.handle(t.parseConfig({type: 'native'}), play); 451 + expect(transformedPlay.data.artists).eql(primaries.concat(secondaries)); 452 + }); 453 + 454 + }); 455 + 403 456 describe('Conditional Transforming', function () { 404 457 405 458 describe('On Hook', function () { ··· 512 565 513 566 describe('Multiple hook transforms', function () { 514 567 515 - it('Accumulates transforms', async function () { 568 + it('Accumulates transforms within a single stage', async function () { 516 569 component.config = { 517 570 options: { 518 571 playTransform: { ··· 542 595 const play = generatePlay({ track: 'My cool something track' }); 543 596 const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); 544 597 expect(transformed.data.track).equal('My cool final thing track'); 598 + }); 599 + 600 + it('Accumulates transforms across multiple stages', async function () { 601 + component.config = { 602 + options: { 603 + playTransform: { 604 + preCompare: [ 605 + { 606 + title: [ 607 + { 608 + search: "something", 609 + replace: "bar" 610 + } 611 + ] 612 + }, 613 + { 614 + type: 'native' 615 + } 616 + ] 617 + } 618 + } 619 + } 620 + 621 + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); 622 + 623 + component.buildTransformRules(); 624 + const play = generatePlay({ track: 'My cool something track', artists: [str] }); 625 + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); 626 + expect(transformed.data.track).equal('My cool bar track'); 627 + expect(transformed.data.artists).eql(primaries.concat(secondaries)); 545 628 }); 546 629 547 630 });
+17
src/backend/utils/PlayComparisonUtils.ts
··· 3 3 import { buildTrackString } from "../../core/StringUtils.js"; 4 4 import { playObjDataMatch } from "../utils.js"; 5 5 import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js"; 6 + import { RestType } from "ts-json-schema-generator"; 6 7 7 8 8 9 export const metaInvariantTransform = (play: PlayObject): PlayObject => { ··· 31 32 ...play.data, 32 33 playDate: undefined 33 34 } 35 + } 36 + } 37 + 38 + export const playContentInvariantTransform = (play: PlayObject): PlayObject => { 39 + const { 40 + data: { 41 + playDate, 42 + playDateCompleted, 43 + ...rest 44 + } 45 + } = play; 46 + return { 47 + data: { 48 + ...rest 49 + }, 50 + meta: {} 34 51 } 35 52 } 36 53
+3 -173
src/backend/utils/PlayTransformUtils.ts
··· 8 8 ConditionalSearchAndReplaceTerm, 9 9 ExternalMetadataTerm, 10 10 PlayTransformParts, 11 - PlayTransformPartsArray, 12 - PlayTransformPartsConfig, 13 - PlayTransformRules, PlayTransformStage, PlayTransformUserParts, PlayTransformUserStage, SearchAndReplaceTerm, 11 + PlayTransformStage, 12 + SearchAndReplaceTerm, 14 13 STAGE_TYPES, 15 14 StageType, 16 15 StageTypedConfig, ··· 172 171 export interface TransformPlayPartsOptions { 173 172 logger?: () => Logger, 174 173 regex?: SuppliedRegex 175 - } 176 - 177 - export const transformPlayUsingParts = (play: PlayObject, parts: PlayTransformUserParts<ConditionalSearchAndReplaceRegExp>, options?: TransformPlayPartsOptions): PlayObject => { 178 - const { 179 - data: { 180 - track, 181 - artists, 182 - albumArtists, 183 - album 184 - } = {} 185 - } = play; 186 - 187 - const { 188 - logger = () => loggerTest, 189 - regex: { 190 - searchAndReplace = searchAndReplaceFunc, 191 - testMaybeRegex = testMaybeRegexFunc, 192 - } = {}, 193 - } = options || {}; 194 - 195 - const transformedPlayData: Partial<ObjectPlayData> = {}; 196 - 197 - let isTransformed = false; 198 - 199 - if(parts.when !== undefined) { 200 - if(!testWhenConditions(parts.when, play, {testMaybeRegex})) { 201 - return play; 202 - } 203 - } 204 - 205 - const searchAndReplaceMapper = (x: ConditionalSearchAndReplaceRegExp): ConditionalSearchAndReplaceRegExp => ({...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, {testMaybeRegex}) : undefined)}) 206 - 207 - if (parts.title !== undefined && track !== undefined) { 208 - try { 209 - const t = searchAndReplace(track, parts.title.map(x => ({...x, test: (x.when !== undefined ? () => testWhenConditions(x.when, play, {testMaybeRegex}) : undefined)}))); 210 - if (t !== track) { 211 - transformedPlayData.track = t.trim() === '' ? undefined : t; 212 - isTransformed = true; 213 - } 214 - } catch (e) { 215 - logger().warn(new Error(`Failed to transform title: ${track}`, {cause: e})); 216 - } 217 - } 218 - 219 - if (parts.artists !== undefined && artists !== undefined && artists.length > 0) { 220 - const transformedArtists: string[] = []; 221 - let anyArtistTransformed = false; 222 - for (const artist of artists) { 223 - try { 224 - const t = searchAndReplace(artist, parts.artists.map(searchAndReplaceMapper)); 225 - if (t !== artist) { 226 - anyArtistTransformed = true; 227 - isTransformed = true; 228 - } 229 - if (t.trim() !== '') { 230 - transformedArtists.push(t); 231 - } 232 - } catch (e) { 233 - logger().warn(new Error(`Failed to transform artist: ${artist}`, {cause: e})); 234 - transformedArtists.push(artist); 235 - } 236 - } 237 - if (anyArtistTransformed) { 238 - transformedPlayData.artists = transformedArtists; 239 - } 240 - } 241 - 242 - if (parts.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) { 243 - const transformedArtists: string[] = []; 244 - let anyArtistTransformed = false; 245 - for (const artist of albumArtists) { 246 - try { 247 - const t = searchAndReplace(artist, parts.artists.map(searchAndReplaceMapper)); 248 - if (t !== artist) { 249 - anyArtistTransformed = true; 250 - isTransformed = true; 251 - } 252 - if (t.trim() !== '') { 253 - transformedArtists.push(t); 254 - } 255 - } catch (e) { 256 - logger().warn(new Error(`Failed to transform albumArtist: ${artist}`, {cause: e})); 257 - transformedArtists.push(artist); 258 - } 259 - } 260 - if (anyArtistTransformed) { 261 - transformedPlayData.albumArtists = transformedArtists; 262 - } 263 - } 264 - 265 - if (parts.album !== undefined && album !== undefined) { 266 - try { 267 - const t = searchAndReplace(album, parts.album.map(searchAndReplaceMapper)); 268 - if (t !== album) { 269 - isTransformed = true; 270 - transformedPlayData.album = t.trim() === '' ? undefined : t; 271 - } 272 - } catch (e) { 273 - logger().warn(new Error(`Failed to transform album: ${album}`, {cause: e})); 274 - } 275 - } 276 - 277 - if (isTransformed) { 278 - 279 - const transformedPlay = { 280 - ...play, 281 - data: { 282 - ...play.data, 283 - ...transformedPlayData 284 - } 285 - } 286 - 287 - return transformedPlay; 288 - } 289 - 290 - return play; 291 - } 292 - 293 - // export const countRegexes = (rules: PlayTransformRules): number => { 294 - // let rulesCount = 0; 295 - // if(rules.preCompare !== undefined) { 296 - // for(const hookItem of rules.preCompare) { 297 - // rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); 298 - // } 299 - 300 - // } 301 - // if(rules.postCompare !== undefined) { 302 - // for(const hookItem of rules.postCompare) { 303 - // rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); 304 - // } 305 - // } 306 - // if(rules.compare !== undefined) { 307 - // if(rules.compare.existing !== undefined) { 308 - // for(const hookItem of rules.compare.existing) { 309 - // rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); 310 - // } 311 - // } 312 - // if(rules.compare.candidate !== undefined) { 313 - // for(const hookItem of rules.compare.candidate) { 314 - // rulesCount = countRulesInParts(hookItem) + countWhens(hookItem.when); 315 - // } 316 - // } 317 - // } 318 - // return rulesCount; 319 - // } 320 - 321 - const countWhens = (when: WhenConditionsConfig | undefined): number => { 322 - if(when === undefined) { 323 - return 0; 324 - } 325 - return when.reduce((acc, curr) => { 326 - return acc + Object.keys(curr).length; 327 - },0) 328 - } 329 - 330 - /** 331 - * Counts all rules within title/artist/album + whens WITHIN those rules 332 - * */ 333 - const countRulesInParts = (parts: PlayTransformParts<ConditionalSearchAndReplaceRegExp>): number => { 334 - return Object.entries(parts).reduce((acc: number, entries: [string, ConditionalSearchAndReplaceRegExp[]]) => { 335 - let curr = acc; 336 - for(const rule of (entries[1] ?? [])) { 337 - curr++; 338 - if(typeof rule !== 'string' && rule.when !== undefined) { 339 - curr += countWhens(rule.when); 340 - } 341 - } 342 - return curr; 343 - }, 0) 344 - } 174 + }