+33 -7

src/backend/common/AbstractComponent.ts

Reviewed

··· 8 8 import { buildPlayHumanDiffable, buildTrackString } from "../../core/StringUtils.js"; 9 9 import { CommonClientConfig } from "./infrastructure/config/client/index.js"; 10 10 import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; 11 11 - import { mergeSimpleError, SkipTransformStageError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; 11 11 + import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; 12 12 import { 13 13 PlayTransformRules, 14 14 StageConfig, ··· 24 24 import chalk from 'chalk'; 25 25 import { isDebugMode } from "../utils.js"; 26 26 import { findCauseByReference } from "../utils/ErrorUtils.js"; 27 27 + import { hashObject } from "../utils/StringUtils.js"; 28 28 + import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js"; 29 29 + import { MSCache } from "./Cache.js"; 27 30 28 31 export default abstract class AbstractComponent extends AbstractInitializable { 29 32 ··· 32 35 transformRules: PlayTransformRules = {}; 33 36 regexCache!: ReturnType<typeof cacheFunctions>; 34 37 protected transformManager: TransformerManager; 38 38 + protected cache: MSCache; 35 39 36 40 protected constructor(config: CommonClientConfig | CommonSourceConfig) { 37 41 super(config); 38 42 this.transformManager = getRoot().items.transformerManager; 43 43 + this.cache = getRoot().items.cache(); 39 44 } 40 45 41 46 protected postCache(): Promise<void> { ··· 185 190 return play; 186 191 } 187 192 193 193 + const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode(); 194 194 + 195 195 + const transformHash = `playTransform-${hashObject(hook)}-${hashObject(metaInvariantTransform(play))}`; 196 196 + const cachedTransformPlay = await this.cache.cacheTransform.get<PlayObject>(transformHash); 197 197 + if(cachedTransformPlay !== undefined) { 198 198 + // if(shouldLog) { 199 199 + // logger.debug(`Used cached Transform for => ${buildTrackString(play)}`); 200 200 + // } 201 201 + //logger.debug(`Cache hit ${transformHash}`); 202 202 + return cachedTransformPlay; 203 203 + } 204 204 + 188 205 logger.debug(`Transform start for => ${buildTrackString(play)}`); 189 206 let transformedPlay: PlayObject = play; 190 207 let transformHistory: TransformResult[] = []; ··· 199 216 200 217 let newTransformedPlay: PlayObject, 201 218 stageName: string = 'Unnamed', 202 202 - err: Error; 219 219 + err: Error, 220 220 + cacheOk: boolean = true; 203 221 try { 204 222 [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId); 205 223 } catch (e) { ··· 210 228 } 211 229 212 230 if(err !== undefined) { 231 231 + const merged = mergeSimpleError(err); 213 232 const skipError = findCauseByReference(err, SkipTransformStageError); 214 233 if(skipError !== undefined) { 215 234 let skipMsg = `Stage '${stageName}' was skipped`; 216 235 if(onSkip === 'stop') { 217 236 skipMsg += ' and will stop transform due to onSkip: stop'; 218 237 } 219 219 - logger.debug(mergeSimpleError(err), skipMsg); 238 238 + logger.debug(merged, skipMsg); 220 239 if(onSkip === 'stop') { 221 240 break; 222 241 } 223 242 } else if(onFailure === 'continue') { 224 224 - logger.warn(mergeSimpleError(err), 'A transform encountered an error but continuing due to onFailure: continue'); 243 243 + logger.warn(merged, 'A transform encountered an error but continuing due to onFailure: continue'); 225 244 } else { 226 226 - logger.error(mergeSimpleError(err), 'Transform encountered an error'); 245 245 + const reqError = findCauseByReference(err, StagePrerequisiteError); 246 246 + if(reqError !== undefined) { 247 247 + logger.warn(merged, 'Transform could not be completed due to prerequisite failure'); 248 248 + } else { 249 249 + logger.error(merged, 'Transform encountered an error'); 250 250 + cacheOk = false; 251 251 + } 252 252 + 227 253 if(!failureReturnPartial) { 228 254 // rewind to original play so we don't return partial transform 229 255 transformedPlay = play; ··· 245 271 break; 246 272 } 247 273 } 248 248 - 249 249 - const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode(); 250 274 251 275 if(transformedPlay.meta.transforms === undefined) { 252 276 transformedPlay.meta.transforms = { ··· 295 319 } 296 320 const previousHistory = transformedPlay.meta.transforms[hookType] ?? []; 297 321 transformedPlay.meta.transforms[hookType] = [...previousHistory, ...transformHistory]; 322 322 + await this.cache.cacheTransform.set<PlayObject>(transformHash, transformedPlay, '5m'); 323 323 + //logger.debug(`Cache set ${transformHash}`) 298 324 return transformedPlay; 299 325 } catch (e) { 300 326 logger.warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e}));

+1

src/backend/common/infrastructure/Transform.ts

Reviewed

··· 60 60 61 61 export interface StageConfig extends StageTypedConfig, Whennable, Partial<FlowControl> { 62 62 name?: string 63 63 + stageHash?: string 63 64 } 64 65 65 66 export interface UntypedStageConfig extends Whennable, Partial<FlowControl> {

+7 -1

src/backend/common/transforms/AbstractTransformer.ts

Reviewed

··· 7 7 import { Cacheable } from "cacheable"; 8 8 import { hashObject } from "../../utils/StringUtils.js"; 9 9 import { playContentInvariantTransform } from "../../utils/PlayComparisonUtils.js"; 10 10 - import { isSimpleError, SkipTransformStageError } from "../errors/MSErrors.js"; 10 10 + import { isSimpleError, SkipTransformStageError, StagePrerequisiteError } from "../errors/MSErrors.js"; 11 11 import { capitalize } from "../../../core/StringUtils.js"; 12 12 13 13 export interface TransformerOptions { ··· 79 79 try { 80 80 await this.handlePreFetch(play, data); 81 81 } catch (e) { 82 82 + if(e instanceof SkipTransformStageError) { 83 83 + await this.cache.set(cacheKey, play, this.config.options?.ttl ?? '15s'); 84 84 + } 82 85 throw new Error('preFetch check did not pass', { cause: e }); 83 86 } 84 87 ··· 93 96 try { 94 97 transformData = await this.handlePostFetch(play, fetchedTransformData, data); 95 98 } catch (e) { 99 99 + if(e instanceof StagePrerequisiteError) { 100 100 + await this.cache.set(cacheKey, play, this.config.options?.ttl ?? '15s'); 101 101 + } 96 102 throw new Error('postFetch did not pass', { cause: e }); 97 103 } 98 104

+4 -4

src/backend/common/transforms/MusicbrainzTransformer.ts

Reviewed

··· 10 10 import { MusicbrainzApiClient, MusicbrainzApiConfig, recordingToPlay, UsingTypes } from "../vendor/musicbrainz/MusicbrainzApiClient.js"; 11 11 import { IRecordingList, IRecordingMatch, MusicBrainzApi } from "musicbrainz-api"; 12 12 import { intersect, isDebugMode, missingMbidTypes, removeUndefinedKeys } from "../../utils.js"; 13 13 - import { SimpleError, SkipTransformStageError } from "../errors/MSErrors.js"; 13 13 + import { SimpleError, SkipTransformStageError, StagePrerequisiteError } from "../errors/MSErrors.js"; 14 14 import { parseArrayFromMaybeString } from "../../utils/StringUtils.js"; 15 15 import clone from "clone"; 16 16 import { Cacheable } from "cacheable"; ··· 559 559 560 560 public async handlePostFetch(play: PlayObject, transformData: IRecordingMSList, stageConfig: MusicbrainzTransformerDataStage): Promise<PlayObject> { 561 561 if(transformData.recordings.length === 0) { 562 562 - throw new SimpleError('No matches returned from Musicbrainz API', {shortStack: true}); 562 562 + throw new StagePrerequisiteError('No matches returned from Musicbrainz API', {shortStack: true}); 563 563 } 564 564 565 565 const { ··· 568 568 569 569 let filteredList: RecordingRankedMatched[] = transformData.recordings.filter(x => x.score >= score); 570 570 if(filteredList.length === 0) { 571 571 - throw new SimpleError(`All ${transformData.count} fetched matches had a score < ${score}, best match was ${transformData.recordings[0].score}`, {shortStack: true}); 571 571 + throw new StagePrerequisiteError(`All ${transformData.count} fetched matches had a score < ${score}, best match was ${transformData.recordings[0].score}`, {shortStack: true}); 572 572 } 573 573 const mergedConfig = Object.assign({}, removeUndefinedKeys({...this.defaults}), removeUndefinedKeys({...stageConfig})); 574 574 filteredList = filterByValidReleaseStatus(filteredList, mergedConfig); ··· 578 578 579 579 580 580 if(filteredList.length === 0) { 581 581 - throw new SimpleError(`All ${transformData.count} recordings were filtered out by allow/deny release config`, {shortStack: true}); 581 581 + throw new StagePrerequisiteError(`All ${transformData.count} recordings were filtered out by allow/deny release config`, {shortStack: true}); 582 582 } 583 583 584 584 filteredList = rankReleasesByPriority(filteredList, mergedConfig);

+3 -1

src/backend/common/transforms/TransformerManager.ts

Reviewed

··· 129 129 throw new Error(`Must be an object with a 'type' property.`); 130 130 } 131 131 const t = this.getTransformerByStage(data); 132 132 - return t.parseConfig(data); 132 132 + const config = t.parseConfig(data); 133 133 + config.stageHash = t.configHash; 134 134 + return config; 133 135 } 134 136 135 137 public async handleStage(data: StageConfig, play: PlayObject, asyncId: string = nanoid(6)): Promise<[PlayObject, string]> {

-1

src/backend/utils/PlayComparisonUtils.ts

Reviewed

··· 3 3 import { buildTrackString } from "../../core/StringUtils.js"; 4 4 import { playObjDataMatch } from "../utils.js"; 5 5 import { comparePlayTemporally, hasAcceptableTemporalAccuracy, TemporalPlayComparisonOptions } from "./TimeUtils.js"; 6 6 - import { RestType } from "ts-json-schema-generator"; 7 6 8 7 9 8 export const metaInvariantTransform = (play: PlayObject): PlayObject => {