foxxmd.dev / multi-scrobbler
[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients docs.multi-scrobbler.app
deezer docker jellyfin koito lastfm listenbrainz maloja mopidy mpris music music-assistant plex scrobble self-hosted spotify subsonic tautulli youtube-music
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fix: Refactor play tranforming to dogfood steps and fix caching #495

* Re-use lifecycle steps/patch to build diffs
* Break out step generation into own function to simplify logic
* Cache steps instead of play to avoid returning a cached play when data is the same #495
* Replace patch object/patching logic with json-diff-ts as its more robust and actually maintained
* Reduces need to stringify everywhere due to lack of cloning in jsonpatchdiff

FoxxMD (Mar 25, 2026, 6:48 PM UTC) 82c0a4fa ccbd5e30

+308 -257
+7 -84
package-lock.json
··· 72 72 "icecast-metadata-stats": "^0.1.12", 73 73 "iso-websocket": "^0.3.0", 74 74 "iti": "^0.7.0", 75 - "jest-diff": "^30.2.0", 75 + "json-diff-ts": "^5.0.0-alpha.2", 76 76 "json5": "^2.2.3", 77 77 "jsondiffpatch": "^0.7.3", 78 78 "jsondiffpatch-react": "^1.0.8", ··· 2568 2568 "axios": "^1.3.4" 2569 2569 } 2570 2570 }, 2571 - "node_modules/@jest/diff-sequences": { 2572 - "version": "30.0.1", 2573 - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", 2574 - "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", 2575 - "license": "MIT", 2576 - "engines": { 2577 - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" 2578 - } 2579 - }, 2580 - "node_modules/@jest/get-type": { 2581 - "version": "30.1.0", 2582 - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", 2583 - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", 2584 - "license": "MIT", 2585 - "engines": { 2586 - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" 2587 - } 2588 - }, 2589 - "node_modules/@jest/schemas": { 2590 - "version": "30.0.5", 2591 - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", 2592 - "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", 2593 - "license": "MIT", 2594 - "dependencies": { 2595 - "@sinclair/typebox": "^0.34.0" 2596 - }, 2597 - "engines": { 2598 - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" 2599 - } 2600 - }, 2601 2571 "node_modules/@joshwooding/vite-plugin-react-docgen-typescript": { 2602 2572 "version": "0.6.4", 2603 2573 "resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.6.4.tgz", ··· 3926 3896 "version": "10.0.2", 3927 3897 "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", 3928 3898 "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", 3929 - "license": "MIT" 3930 - }, 3931 - "node_modules/@sinclair/typebox": { 3932 - "version": "0.34.41", 3933 - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz", 3934 - "integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==", 3935 3899 "license": "MIT" 3936 3900 }, 3937 3901 "node_modules/@sindresorhus/is": { ··· 13525 13489 } 13526 13490 } 13527 13491 }, 13528 - "node_modules/jest-diff": { 13529 - "version": "30.2.0", 13530 - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", 13531 - "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", 13532 - "license": "MIT", 13533 - "dependencies": { 13534 - "@jest/diff-sequences": "30.0.1", 13535 - "@jest/get-type": "30.1.0", 13536 - "chalk": "^4.1.2", 13537 - "pretty-format": "30.2.0" 13538 - }, 13539 - "engines": { 13540 - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" 13541 - } 13542 - }, 13543 - "node_modules/jest-diff/node_modules/ansi-styles": { 13544 - "version": "5.2.0", 13545 - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", 13546 - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", 13547 - "license": "MIT", 13548 - "engines": { 13549 - "node": ">=10" 13550 - }, 13551 - "funding": { 13552 - "url": "https://github.com/chalk/ansi-styles?sponsor=1" 13553 - } 13554 - }, 13555 - "node_modules/jest-diff/node_modules/pretty-format": { 13556 - "version": "30.2.0", 13557 - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", 13558 - "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", 13559 - "license": "MIT", 13560 - "dependencies": { 13561 - "@jest/schemas": "30.0.5", 13562 - "ansi-styles": "^5.2.0", 13563 - "react-is": "^18.3.1" 13564 - }, 13565 - "engines": { 13566 - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" 13567 - } 13568 - }, 13569 - "node_modules/jest-diff/node_modules/react-is": { 13570 - "version": "18.3.1", 13571 - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", 13572 - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", 13573 - "license": "MIT" 13574 - }, 13575 13492 "node_modules/jest-get-type": { 13576 13493 "version": "27.5.1", 13577 13494 "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-27.5.1.tgz", ··· 13682 13599 "version": "3.0.1", 13683 13600 "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", 13684 13601 "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", 13602 + "license": "MIT" 13603 + }, 13604 + "node_modules/json-diff-ts": { 13605 + "version": "5.0.0-alpha.2", 13606 + "resolved": "https://registry.npmjs.org/json-diff-ts/-/json-diff-ts-5.0.0-alpha.2.tgz", 13607 + "integrity": "sha512-7AI6rZNrvqW3Bkmm9Wsq6985MeC8KkK68mgtW7z3aq39QBGMcoyClrbAXlUr6yQ0VbafVtAM+f5b4bAwlxVSAg==", 13685 13608 "license": "MIT" 13686 13609 }, 13687 13610 "node_modules/json-parse-even-better-errors": {
+1 -1
package.json
··· 110 110 "icecast-metadata-stats": "^0.1.12", 111 111 "iso-websocket": "^0.3.0", 112 112 "iti": "^0.7.0", 113 - "jest-diff": "^30.2.0", 113 + "json-diff-ts": "^5.0.0-alpha.2", 114 114 "json5": "^2.2.3", 115 115 "jsondiffpatch": "^0.7.3", 116 116 "jsondiffpatch-react": "^1.0.8",
+180 -139
src/backend/common/AbstractComponent.ts
··· 10 10 import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; 11 11 import { mergeSimpleError, SkipTransformStageError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.js"; 12 12 import { 13 + FLOW_CONTROL_TERM, 13 14 PlayTransformRules, 14 15 StageConfig, 15 16 TRANSFORM_HOOK, 16 17 TransformHook 17 18 } from "./infrastructure/Transform.js"; 18 19 import AbstractInitializable from "./AbstractInitializable.js"; 19 - import play = Simulate.play; 20 20 import TransformerManager from "./transforms/TransformerManager.js"; 21 21 import { getRoot } from "../ioc.js"; 22 22 import { nanoid } from "nanoid"; 23 - import {diffStringsUnified, DiffOptionsColor} from 'jest-diff'; 24 - import chalk from 'chalk'; 25 23 import { isDebugMode } from "../utils.js"; 26 24 import { findCauseByReference } from "../utils/ErrorUtils.js"; 27 25 import { hashObject } from "../utils/StringUtils.js"; 28 26 import { metaInvariantTransform, playContentInvariantTransform } from "../utils/PlayComparisonUtils.js"; 29 27 import { MSCache } from "./Cache.js"; 30 - import { jdiff } from "../../core/DataUtils.js"; 31 - import ConsoleFormatter from "jsondiffpatch/formatters/console"; 28 + import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.js"; 32 29 import clone from "clone"; 33 - 34 - const console = new ConsoleFormatter(); 30 + import { loggerNoop } from "./MaybeLogger.js"; 35 31 36 32 export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager }; 37 33 ··· 169 165 return partArr.map(x => this.transformManager.parseTransformerConfig(x)); 170 166 } 171 167 172 - public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean) => { 168 + public transformPlay = async (play: PlayObject, hookType: TransformHook, log?: boolean | 'all') => { 173 169 174 170 175 171 const asyncId = nanoid(6); ··· 200 196 const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode(); 201 197 202 198 const transformHash = `playTransform-${hashObject(hook)}-${hashObject(playContentInvariantTransform(play))}`; 203 - const cachedTransformPlay = await this.cache.cacheTransform.get<PlayObject>(transformHash); 204 - if(cachedTransformPlay !== undefined) { 205 - logger.trace(`Cache hit ${transformHash}`); 206 - return cachedTransformPlay; 199 + const cachedSteps = await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash); 200 + if(cachedSteps !== undefined) { 201 + logger.trace(`Cache hit for Steps => ${transformHash}`); 202 + //return cachedTransformPlay; 207 203 } 208 204 209 205 logger.debug(`Transform start for => ${buildTrackString(play)}`); 210 - let transformedPlay: PlayObject = play; 211 - let transformHistory: TransformResult[] = []; 212 - let shouldBreak = false; 206 + let transformedPlay: PlayObject = clone(play); 213 207 let cacheOk: boolean = true; 214 - for(const hookItem of hook) { 208 + const steps: LifecycleStep[] = []; 215 209 216 - const { 217 - onSuccess = 'continue', 218 - onFailure = 'stop', 219 - onSkip = 'continue', 220 - failureReturnPartial = false 221 - } = hookItem; 210 + const opts = { 211 + logger, 212 + asyncId 213 + } 222 214 223 - const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier()); 224 - const step: LifecycleStep = existingStepIndex !== -1 ? transformedPlay.meta.lifecycle.steps[existingStepIndex] : { 225 - name: hookType, 226 - source: this.getIdentifier(), 215 + if(cachedSteps !== undefined) { 216 + // don't re-cache cached steps 217 + cacheOk = false; 218 + // only patch play if steps didn't end in a failure that we are okay with returning partial from 219 + let shouldTransform = true; 220 + const lastCachedStep = cachedSteps[cachedSteps.length - 1]; 221 + if(lastCachedStep.error !== undefined && lastCachedStep.flowKnownState !== 'skip' && !lastCachedStep.returnPartial) { 222 + shouldTransform = false; 227 223 } 228 224 229 - let newTransformedPlay: PlayObject, 230 - stageName: string = 'Unnamed', 231 - err: Error; 232 - try { 233 - [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, transformedPlay, asyncId); 234 - } catch (e) { 235 - err = e; 236 - if(e instanceof StageTransformError) { 237 - stageName = e.stageName; 225 + for(const s of cachedSteps) { 226 + steps.push({...s, cached: true}); 227 + if(shouldTransform && s.patch !== undefined) { 228 + transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch); 238 229 } 239 230 } 240 - 241 - if(err !== undefined) { 242 - const merged = mergeSimpleError(err); 243 - step.error = merged; 244 - 245 - const skipError = findCauseByReference(err, SkipTransformStageError); 246 - if(skipError !== undefined) { 247 - let skipMsg = `Stage '${stageName}' was skipped`; 248 - step.flowResult = onSkip; 249 - step.flowKnownState = 'skip'; 250 - 251 - if(onSkip === 'stop') { 252 - skipMsg += ' and will stop transform due to onSkip: stop'; 253 - } 254 - step.flowReason = skipMsg; 255 - 256 - logger.debug(merged, skipMsg); 257 - if(onSkip === 'stop') { 258 - shouldBreak = true; 259 - } 260 - } else { 261 - step.flowResult = onFailure; 262 - let reason: string; 263 - 264 - const reqError = findCauseByReference(err, StagePrerequisiteError); 265 - if(reqError !== undefined) { 266 - reason = 'Transform could not be completed due to prerequisite failure'; 267 - step.flowKnownState = 'prereq'; 268 - } else { 269 - reason = 'Transform encountered an error'; 270 - cacheOk = false; 271 - } 272 - 273 - if(onFailure === 'continue') { 274 - reason += ' but will continue due to onFailure: continue'; 275 - logger.warn(merged, reason); 231 + } else { 232 + for(const hookItem of hook) { 233 + const [step, stepPlay] = await this.generateStepFromStage(transformedPlay, hookItem, hookType, opts); 234 + steps.push(step); 235 + cacheOk = step.error === undefined || (step.error !== undefined && step.flowKnownState !== undefined); 236 + if(step.flowResult === FLOW_CONTROL_TERM.stop) { 237 + if(step.error !== undefined && step.flowKnownState !== 'skip' && !step.returnPartial) { 238 + // revert to original play but keep steps for paper trail 239 + transformedPlay = play; 240 + transformedPlay.meta.lifecycle.steps = steps; 276 241 } else { 277 - logger[reqError !== undefined ? 'warn' : 'error'](merged, reason); 278 - 279 - if(!failureReturnPartial) { 280 - // rewind to original play so we don't return partial transform 281 - transformedPlay = play; 282 - transformHistory = []; 283 - } else if(transformHistory.length > 0) { 284 - cacheOk = true; 285 - step.flowReason += ' | Preserving play transformations up to this point due to failureReturnPartial=true' 286 - } 287 - shouldBreak = true; 242 + transformedPlay = stepPlay; 288 243 } 289 - 290 - step.flowReason = reason; 291 - } 292 - } else { 293 - // reset cache state 294 - // in the event it was false due to previous step error but onFailure: continue 295 - cacheOk = true; 296 - step.flowResult = onSuccess; 297 - 298 - if(!deepEqual(play, newTransformedPlay)) { 299 - const o = JSON.parse(JSON.stringify(play)); 300 - const t = JSON.parse(JSON.stringify(newTransformedPlay)); 301 - const patch = jdiff.diff(o,t); 302 - step.patch = patch; 303 - } 304 - 305 - if(newTransformedPlay.meta.lifecycleInputs?.length > 0) { 306 - step.inputs = clone(newTransformedPlay.meta.lifecycleInputs) 307 - } else if(transformedPlay.meta.lifecycleInputs?.length > 0) { 308 - step.inputs = clone(transformedPlay.meta.lifecycleInputs) 244 + break; 245 + } else { 246 + transformedPlay = stepPlay; 309 247 } 310 - 311 - delete transformedPlay.meta.lifecycleInputs; 312 - 313 - // for logging only 314 - transformHistory.push({ 315 - type: hookItem.type, 316 - name: stageName, 317 - play: newTransformedPlay.data 318 - }); 319 - transformedPlay = newTransformedPlay; 320 248 } 249 + } 321 250 322 - if(err === undefined && onSuccess === 'stop') { 323 - logger.debug(`Stopping transform due to onSuccess: stop`); 324 - shouldBreak = true; 251 + const historyToDiff: {name: string, data?: PlayData}[] = [ 252 + {name: 'Pre Transform', data: play.data} 253 + ]; 254 + if(steps.length > 0 && transformedPlay.meta?.lifecycle?.steps === undefined) { 255 + const { 256 + meta: { 257 + lifecycle: { 258 + steps = [], 259 + ...lifecycleRest 260 + }, 261 + } = {} 262 + } = transformedPlay; 263 + transformedPlay.meta.lifecycle = { 264 + ...lifecycleRest, 265 + steps 325 266 } 326 - 267 + } 268 + steps.forEach((s, index) => { 269 + const existingStepIndex = transformedPlay.meta.lifecycle.steps.findIndex(x => x.name === 'hookType' && x.source === this.getIdentifier()); 327 270 if(existingStepIndex !== -1) { 328 - transformedPlay.meta.lifecycle.steps[existingStepIndex] = step; 271 + transformedPlay.meta.lifecycle.steps[existingStepIndex] = s; 329 272 } else { 330 - transformedPlay.meta.lifecycle.steps.push(step); 273 + transformedPlay.meta.lifecycle.steps.push(s); 331 274 } 332 275 333 - if(shouldBreak) { 334 - break; 276 + if(shouldLog === 'all') { 277 + if(s.patch === undefined) { 278 + historyToDiff.push({name: `${s.source}-${s.name}`}); 279 + } else { 280 + const patched = patchObject(historyToDiff[historyToDiff.length - 1].data, s.patch); 281 + historyToDiff.push({name: `${s.source}-${s.name}${s.cached ? ' (Cached)' : ''}`, data: patched}); 282 + } 335 283 } 336 - } 337 - 284 + }); 338 285 if(shouldLog !== false) { 339 - if(transformHistory.length === 0) { 286 + if(steps.filter(x => x.patch !== undefined).length === 0) { 340 287 logger.debug('Transform Diff: No Change'); 341 288 } else { 342 - const historyToDiff: {name: string, data: PlayData}[] = [ 343 - {name: 'Original', data: play.data} 344 - ]; 345 - if(shouldLog === true) { 346 - const last = transformHistory[transformHistory.length - 1]; 347 - historyToDiff.push({name: `${last.type}-${last.name}`, data: last.play}); 348 - } else { 349 - for(const t of transformHistory) { 350 - historyToDiff.push({name: `${t.type}-${t.name}`, data: t.play}) 351 - } 352 - } 353 289 const diffs: string[] = []; 290 + 354 291 historyToDiff.forEach((curr, index) => { 355 292 if(index === 0) { 356 293 return; 357 294 } 358 295 const last = historyToDiff[index - 1]; 359 - if(deepEqual(last.data, curr.data)) { 296 + if(curr.data === undefined) { 360 297 diffs.push(`${last.name} => ${curr.name} -- No Change`); 361 298 } else { 362 - const unchanged = index - 1 === 0 ? last.data : undefined; 363 - 364 - const formattedDiff = console.format(jdiff.diff(last.data, curr.data), unchanged); 299 + const formattedDiff = diffObjectsConsoleOutput(last.data, curr.data); 365 300 diffs.push(`${last.name} => ${curr.name}\n${formattedDiff}`); 366 301 } 367 302 }); 303 + 304 + if(shouldLog === true || steps.filter(x => x.patch !== undefined).length > 2) { 305 + const finalData = transformedPlay.data; 306 + const formattedDiff = diffObjectsConsoleOutput(play.data, finalData, true); 307 + diffs.push(`Original => Final\n${formattedDiff}`); 308 + } 309 + 368 310 logger.debug(`Transform Diff\n${diffs.join('\n')}`); 369 311 } 370 312 } 371 313 372 314 if(cacheOk) { 373 - await this.cache.cacheTransform.set<PlayObject>(transformHash, transformedPlay, '10m'); 315 + await this.cache.cacheTransform.set<LifecycleStep[]>(transformHash, steps, '10m'); 374 316 } 375 317 376 318 return transformedPlay; ··· 378 320 logger.warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e})); 379 321 return play; 380 322 } 323 + } 324 + 325 + protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } = {}): Promise<[LifecycleStep, PlayObject]> => { 326 + const { 327 + onSuccess = 'continue', 328 + onFailure = 'stop', 329 + onSkip = 'continue', 330 + failureReturnPartial = false 331 + } = hookItem; 332 + 333 + const { 334 + logger = loggerNoop, 335 + asyncId = nanoid(6) 336 + } = opts; 337 + 338 + const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}` 339 + const existingStepIndex = playTruth.meta.lifecycle.steps.findIndex(x => x.name === stepName && x.source === this.getIdentifier()); 340 + const step: LifecycleStep = existingStepIndex !== -1 ? playTruth.meta.lifecycle.steps[existingStepIndex] : { 341 + name: stepName, 342 + source: this.getIdentifier(), 343 + } 344 + 345 + let newTransformedPlay: PlayObject, 346 + stageName: string = 'Unnamed', 347 + err: Error; 348 + try { 349 + [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, asyncId); 350 + newTransformedPlay = clone(newTransformedPlay); 351 + } catch (e) { 352 + err = e; 353 + if (e instanceof StageTransformError) { 354 + stageName = e.stageName; 355 + } 356 + } 357 + 358 + if (err !== undefined) { 359 + const merged = mergeSimpleError(err); 360 + step.error = merged; 361 + 362 + const skipError = findCauseByReference(err, SkipTransformStageError); 363 + if (skipError !== undefined) { 364 + let skipMsg = `Stage '${stageName}' was skipped`; 365 + step.flowResult = onSkip; 366 + step.flowKnownState = 'skip'; 367 + 368 + if (onSkip === 'stop') { 369 + skipMsg += ' and will stop transform due to onSkip: stop'; 370 + } 371 + step.flowReason = skipMsg; 372 + 373 + logger.debug(merged, skipMsg); 374 + } else { 375 + step.flowResult = onFailure; 376 + let reason: string; 377 + 378 + const reqError = findCauseByReference(err, StagePrerequisiteError); 379 + if (reqError !== undefined) { 380 + reason = 'Transform could not be completed due to prerequisite failure'; 381 + step.flowKnownState = 'prereq'; 382 + } else { 383 + reason = 'Transform encountered an error'; 384 + } 385 + 386 + if (onFailure === 'continue') { 387 + reason += ' but will continue due to onFailure: continue'; 388 + logger.warn(merged, reason); 389 + } else { 390 + if (failureReturnPartial) { 391 + reason += ' | Preserving play transformations up to this point due to failureReturnPartial=true'; 392 + step.returnPartial = true; 393 + } 394 + 395 + logger[reqError !== undefined ? 'warn' : 'error'](merged, reason); 396 + } 397 + step.flowReason = reason; 398 + 399 + return [step, playTruth]; 400 + } 401 + } else { 402 + step.flowResult = onSuccess; 403 + 404 + if (!deepEqual(playTruth.data, newTransformedPlay.data)) { 405 + const o = JSON.parse(JSON.stringify(playTruth.data)); 406 + const t = JSON.parse(JSON.stringify(newTransformedPlay.data)); 407 + const patch = diffObjects(o, t); // jdiff.diff(o, t); 408 + step.patch = patch; 409 + } 410 + 411 + if (newTransformedPlay.meta.lifecycleInputs?.length > 0) { 412 + step.inputs = clone(newTransformedPlay.meta.lifecycleInputs) 413 + } else if (playTruth.meta.lifecycleInputs?.length > 0) { 414 + logger.warn({ label: `${hookItem.type} - ${hookItem.name}` }, `Should only be adding inputs to transformed play!`); 415 + step.inputs = clone(playTruth.meta.lifecycleInputs) 416 + } 417 + 418 + delete newTransformedPlay.meta.lifecycleInputs; 419 + } 420 + 421 + return [step, newTransformedPlay]; 381 422 } 382 423 }
+4
src/backend/common/infrastructure/Transform.ts
··· 50 50 } 51 51 52 52 export type FlowControlTerm = 'continue' | 'stop' 53 + export const FLOW_CONTROL_TERM = { 54 + continue: 'continue', 55 + stop: 'stop' 56 + } as const satisfies Record<string, FlowControlTerm>; 53 57 54 58 export interface FlowControl { 55 59 onSuccess: FlowControlTerm
+45
src/backend/tests/component/transformers.test.ts
··· 16 16 import { TransformerCommonConfig } from "../../../core/Atomic.js"; 17 17 import TransformerManager from "../../common/transforms/TransformerManager.js"; 18 18 import { transientCache } from "../utils/CacheTestUtils.js"; 19 + import dayjs from "dayjs"; 20 + import clone from "clone"; 19 21 20 22 chai.use(asPromised); 21 23 ··· 634 636 const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare); 635 637 expect(transformed.data.track).equal('My cool bar track'); 636 638 expect(transformed.data.artists).eql(primaries.concat(secondaries)); 639 + }); 640 + 641 + }); 642 + 643 + describe("Stage Caching", function () { 644 + 645 + it('Re-uses steps without modifying other Play properties', async function () { 646 + component.config = { 647 + options: { 648 + playTransform: { 649 + preCompare: [ 650 + { 651 + name: "barChange", 652 + title: [ 653 + { 654 + search: "something", 655 + replace: "bar" 656 + } 657 + ] 658 + }, 659 + { 660 + type: 'native' 661 + } 662 + ] 663 + } 664 + } 665 + } 666 + 667 + const [str, primaries, secondaries] = generateArtistsStr({primary: {max: 3, ambiguousJoinedNames: true, trailingAmpersand: true, finalJoiner: false}}); 668 + 669 + component.buildTransformRules(); 670 + const play = generatePlay({ track: 'My cool something track', artists: [str], playDate: dayjs().subtract(10, 'm') }); 671 + const transformed = await component.transformPlay(play, TRANSFORM_HOOK.preCompare, 'all'); 672 + expect(transformed.data.track).equal('My cool bar track'); 673 + expect(transformed.data.artists).eql(primaries.concat(secondaries)); 674 + 675 + const cachablePlay = clone(play); 676 + const laterDate = dayjs().subtract(5, 'm'); 677 + cachablePlay.data.playDate = laterDate; 678 + const cacheTransformed = await component.transformPlay(cachablePlay, TRANSFORM_HOOK.preCompare, 'all'); 679 + expect(cacheTransformed.data.track).equal('My cool bar track'); 680 + expect(cacheTransformed.data.artists).eql(primaries.concat(secondaries)); 681 + expect(cacheTransformed.data.playDate.isSame(cachablePlay.data.playDate)); 637 682 }); 638 683 639 684 });
+5 -5
src/client/components/JsonDiff.tsx
··· 1 1 import React, { ComponentProps } from 'react'; 2 2 import JsonDiffReact from 'jsondiffpatch-react'; 3 3 import { MSErrorBoundary } from './ErrorBoundary'; 4 - import { Delta } from 'jsondiffpatch'; 5 4 import { MarkOptional } from 'ts-essentials'; 6 - import { jdiff } from '../../core/DataUtils'; 5 + import { patchObject } from '../../core/DataUtils'; 7 6 import './JsonDiff.css'; 7 + import { IJsonDelta } from 'json-diff-ts'; 8 8 9 9 type DiffableVal = [] | object | string; 10 10 ··· 18 18 } 19 19 20 20 export type JsonDiffPatchProps = MarkOptional<JsonDiffReactProps, 'right'> & { 21 - diff?: Delta | object 21 + diff?: IJsonDelta 22 22 } 23 23 24 24 export const JsonDiffPatch = (props: JsonDiffPatchProps) => { ··· 31 31 const detachedLeft = JSON.parse(JSON.stringify(left)); 32 32 let realRight: DiffableVal; 33 33 if (right !== undefined) { 34 - realRight = JSON.parse(JSON.stringify(right)); 34 + realRight = structuredClone(right); 35 35 } else if (diff !== undefined) { 36 - realRight = jdiff.patch(JSON.parse(JSON.stringify(left)), diff as Delta) as DiffableVal; 36 + realRight = patchObject(left, diff); 37 37 } else { 38 38 throw new Error(`must provide either 'right' or 'diff'`); 39 39 }
+6 -6
src/client/components/TransformSteps.tsx
··· 12 12 import { MdMusicNote } from "react-icons/md"; 13 13 import { ChakraCodeBlockShort, ChakraPlainBlockShort } from "./CodeBlock"; 14 14 import { JsonDiffPatch } from "./JsonDiff"; 15 - import { jdiff } from "../../core/DataUtils"; 15 + import { jdiff, patchObject } from "../../core/DataUtils"; 16 16 import { MSCollapsible, MSCollapsibleExternalProps } from "./MSCollapsible"; 17 17 import { Muted } from "./Typography"; 18 18 ··· 23 23 24 24 const diffElements = (original: JsonPlayObject, steps: LifecycleStep[]): [JSX.Element[], JsonPlayObject?] => { 25 25 26 - let currentPlay: JsonPlayObject = JSON.parse(JSON.stringify(original)); 26 + let currentPlay: JsonPlayObject = structuredClone(original); // JSON.parse(JSON.stringify(original)); 27 27 let patchFailed = false; 28 28 29 29 const diffElements: JSX.Element[] | null = []; ··· 49 49 diffElements.push(<ChakraCodeBlockShort key={`diffblockfallback-${index}`} title="Diff Patch" code={patch} />); 50 50 continue; 51 51 } 52 - let left: JsonPlayObject = JSON.parse(JSON.stringify(currentPlay)); 52 + let left: JsonPlayObject = structuredClone(currentPlay); // JSON.parse(JSON.stringify(currentPlay)); 53 53 left.data.meta = { 54 54 ...(left.data.meta ?? {}), 55 55 brainz: { ··· 64 64 } 65 65 66 66 try { 67 - currentPlay = jdiff.patch(currentPlay, patch) as JsonPlayObject; 67 + currentPlay.data = patchObject(currentPlay.data, patch)// jdiff.patch(currentPlay, patch) as JsonPlayObject; 68 68 diffElements.push( 69 - <ChakraPlainBlockShort title="Play Diff" key={`diffblock-${index}`} code={left}> 70 - <JsonDiffPatch key={`diff-${index}`} left={left} right={JSON.parse(JSON.stringify(currentPlay))} /> 69 + <ChakraPlainBlockShort title="Play Diff" key={`diffblock-${index}`} code={left.data}> 70 + <JsonDiffPatch key={`diff-${index}`} left={left.data} right={currentPlay.data} /> 71 71 </ChakraPlainBlockShort> 72 72 ) 73 73 } catch (e) {
+3 -1
src/core/Atomic.ts
··· 7 7 import { ErrorObject } from "serialize-error"; 8 8 import { PlayPlatformIdStr } from "../backend/common/infrastructure/Atomic.js"; 9 9 import { FlowControlTerm } from "../backend/common/infrastructure/Transform.js"; 10 + import { IJsonDelta } from "json-diff-ts"; 10 11 11 12 export interface SourceStatusData { 12 13 status: string; ··· 311 312 name: string 312 313 source: string 313 314 cached?: boolean 315 + returnPartial?: boolean 314 316 flowResult?: FlowControlTerm 315 317 flowReason?: string 316 318 flowKnownState?: 'skip' | 'prereq' 317 319 error?: ErrorLike 318 - patch?: Delta 320 + patch?: IJsonDelta 319 321 inputs?: LifecycleInput[] 320 322 } 321 323
+36
src/core/DataUtils.ts
··· 1 1 import { create as diffCreate } from "jsondiffpatch"; 2 2 import { numberFormatOptions } from './Atomic.js'; 3 + import { diffDelta, applyDelta, IJsonDelta, DeltaOptions } from 'json-diff-ts'; 4 + // may want to return to this one day 5 + // but currently the jsondiffpatch formatter is the best console/ansi diff output for humans :( 6 + //import {DiffOptions, DiffOptionsColor, diff as jestDiff} from 'jest-diff'; 7 + import chalk from 'chalk'; 8 + import clone from "clone"; 9 + import ConsoleFormatter from "jsondiffpatch/formatters/console"; 3 10 11 + const console = new ConsoleFormatter(); 4 12 5 13 export const jdiff = diffCreate({ 6 14 propertyFilter(name, context) { ··· 9 17 cloneDiffValues: true 10 18 //omitRemovedValues: true 11 19 }); 20 + 21 + const diffOptions: DeltaOptions = { 22 + /*arrayIdentityKeys: {artists: '$value'},*/ 23 + reversible: false, 24 + keysToSkip: ['playDate','playDateCompleted','listenRanges'] 25 + }; 26 + 27 + export const diffObjects = (a: object, b: object) => { 28 + return diffDelta(a, b, diffOptions); 29 + } 30 + 31 + export const patchObject = <T>(a: T, b: IJsonDelta): T => { 32 + return applyDelta(clone(a), b); 33 + } 34 + 35 + // const jestDiffOptions: DiffOptions = { 36 + // aAnnotation: 'Old', 37 + // bAnnotation: 'New', 38 + // aColor: chalk.red, 39 + // bColor: chalk.green 40 + // } 41 + 42 + export const diffObjectsConsoleOutput = (a: object, b: object, showUnchanged: boolean = false) => { 43 + //return jestDiff(a, b, jestDiffOptions); 44 + 45 + const left = JSON.parse(JSON.stringify(a)); 46 + return console.format(jdiff.diff(left, JSON.parse(JSON.stringify(b))), showUnchanged ? left : undefined); 47 + } 12 48 13 49 export const formatNumber = (val: number | string, options?: numberFormatOptions) => { 14 50 const {
+2 -2
src/core/tests/utils/fixtures.ts
··· 8 8 import { generateBrainz, generateMbid, generatePlay, GeneratePlayOpts, generatePlays } from '../../PlayTestUtils.js'; 9 9 import { lifecyclelessInvariantTransform } from '../../PlayUtils.js'; 10 10 import clone from 'clone'; 11 - import { jdiff } from '../../DataUtils.js'; 11 + import { diffObjects } from '../../DataUtils.js'; 12 12 import { existingScrobble } from '../../../backend/utils/PlayComparisonUtils.js'; 13 13 import { UpstreamError } from '../../../backend/common/errors/UpstreamError.js'; 14 14 import { playToListenPayload } from '../../../backend/common/vendor/listenbrainz/lzUtils.js'; ··· 312 312 }); 313 313 } 314 314 315 - step.patch = jdiff.diff(play, modifiedPlay); 315 + step.patch = diffObjects(play.data, modifiedPlay.data);// jdiff.diff(play, modifiedPlay); 316 316 317 317 return [step, modifiedPlay]; 318 318 }
+12 -12
src/stories/ActivityTimeline.stories.tsx
··· 8 8 import { generateJsonPlays } from "../core/PlayTestUtils.js"; 9 9 import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; 10 10 import { examplePlay, lastfmErrorExample } from "./storyUtils.js"; 11 - import { generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js"; 11 + import { asJsonPlayObject, generatePlayWithLifecycle, playWithLifecycleScrobble } from "../core/tests/utils/fixtures.js"; 12 12 13 13 // More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export 14 14 const meta = preview.meta({ ··· 35 35 export const ActivityTimelineStory = meta.story({ 36 36 loaders: [ 37 37 async () => { 38 - const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle( 38 + const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle( 39 39 { 40 40 lifecycleSteps: { 41 41 preCompare: 1, 42 42 postCompare: 1, 43 43 } 44 44 } 45 - )) 45 + ))) 46 46 return {play: scrobbleError}; 47 47 } 48 48 ], ··· 51 51 export const ScrobbleError = meta.story({ 52 52 loaders: [ 53 53 async () => { 54 - const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true}); 54 + const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true})); 55 55 return {play: scrobbleError}; 56 56 } 57 57 ], ··· 60 60 export const TransformError = meta.story({ 61 61 loaders: [ 62 62 async () => { 63 - const scrobbleError = await generatePlayWithLifecycle({ 63 + const scrobbleError = asJsonPlayObject(await generatePlayWithLifecycle({ 64 64 lifecycleSteps: { 65 65 preCompare: 2, 66 66 postCompare: [false], 67 67 } 68 - }); 68 + })); 69 69 return {play: scrobbleError}; 70 70 } 71 71 ], ··· 74 74 export const TransformSkip = meta.story({ 75 75 loaders: [ 76 76 async () => { 77 - const play = await generatePlayWithLifecycle({ 77 + const play = asJsonPlayObject(await generatePlayWithLifecycle({ 78 78 lifecycleSteps: { 79 79 preCompare: [true, 'skipped', true], 80 80 } 81 - }); 81 + })); 82 82 return {play}; 83 83 } 84 84 ], ··· 87 87 export const TransformPrereq = meta.story({ 88 88 loaders: [ 89 89 async () => { 90 - const play = await generatePlayWithLifecycle({ 90 + const play = asJsonPlayObject(await generatePlayWithLifecycle({ 91 91 lifecycleSteps: { 92 92 preCompare: [true, 'prereq'], 93 93 } 94 - }); 94 + })); 95 95 return {play}; 96 96 } 97 97 ], ··· 100 100 export const TransformStop = meta.story({ 101 101 loaders: [ 102 102 async () => { 103 - const play = await generatePlayWithLifecycle({ 103 + const play = asJsonPlayObject(await generatePlayWithLifecycle({ 104 104 lifecycleSteps: { 105 105 preCompare: [true, 'stop'], 106 106 } 107 - }); 107 + })); 108 108 return {play}; 109 109 } 110 110 ],
+5 -5
src/stories/List.stories.tsx
··· 8 8 import { generateJsonPlays, normalizePlays } from "../core/PlayTestUtils.js"; 9 9 import { ErrorLike, JsonPlayObject } from "../core/Atomic.js"; 10 10 import {examplePlay, lastfmErrorExample} from './storyUtils.js'; 11 - import {playWithLifecycleScrobble, generatePlayWithLifecycle} from '../core/tests/utils/fixtures' 11 + import {playWithLifecycleScrobble, generatePlayWithLifecycle, asJsonPlayObject} from '../core/tests/utils/fixtures' 12 12 import { generateArray } from "../core/DataUtils.js"; 13 13 import dayjs from "dayjs"; 14 14 ··· 55 55 export const List = meta.story({ 56 56 loaders: [ 57 57 async () => { 58 - const queued = normalizePlays(generateArray(7,() => generatePlayWithLifecycle()), {endDate: dayjs()}).map(x => ({play: x, status: 'queued'})); 58 + const queued = normalizePlays(generateArray(7,() => generatePlayWithLifecycle()), {endDate: dayjs()}).map(x => ({play: asJsonPlayObject(x), status: 'queued'})); 59 59 60 - const scrobbled = await playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}})); 61 - const scrobbleError = await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true}); 60 + const scrobbled = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}}))); 61 + const scrobbleError = asJsonPlayObject(await playWithLifecycleScrobble(generatePlayWithLifecycle(), {error: true})); 62 62 63 63 const promisedScrobbled = generateArray(10,() => playWithLifecycleScrobble(generatePlayWithLifecycle({lifecycleSteps: {preCompare: [true, 'skipped', true]}}))); 64 64 const promised = await Promise.all(promisedScrobbled); 65 - const yesterdayScrobbled = normalizePlays(promised, {endDate: dayjs().subtract(1, 'd').subtract(100, 'm')}).map((x) => ({play: x, status: 'scrobbled'})); 65 + const yesterdayScrobbled = normalizePlays(promised, {endDate: dayjs().subtract(1, 'd').subtract(100, 'm')}).map((x) => ({play: asJsonPlayObject(x), status: 'scrobbled'})); 66 66 return {data: [ 67 67 ...queued, 68 68 {play: scrobbled, status: 'scrobbled'},
+2 -2
src/stories/TransformSteps.stories.tsx
··· 8 8 import { generateJsonPlays } from "../core/PlayTestUtils.js"; 9 9 import { ErrorLike, JsonPlayObject, PlayLifecycle } from "../core/Atomic.js"; 10 10 import { examplePlay, lastfmErrorExample } from "./storyUtils.js"; 11 - import {generatePlayWithLifecycle} from '../core/tests/utils/fixtures' 11 + import {asJsonPlayObject, generatePlayWithLifecycle} from '../core/tests/utils/fixtures' 12 12 13 13 // More on how to set up stories at: https://storybook.js.org/docs/writing-stories#default-export 14 14 const meta = preview.meta({ ··· 27 27 // Use `fn` to spy on the onClick arg, which will appear in the actions panel once invoked: https://storybook.js.org/docs/essentials/actions#story-args 28 28 }); 29 29 30 - const multiPlay = generatePlayWithLifecycle({lifecycleSteps: { preCompare: 2}}); 30 + const multiPlay = asJsonPlayObject(generatePlayWithLifecycle({lifecycleSteps: { preCompare: 2}})); 31 31 // More on writing stories with args: https://storybook.js.org/docs/writing-stories/args 32 32 export const Multiple = meta.story({ 33 33 args: {