refactor: Refactor cache building to be able to specify both primary and secondary · foxxmd.dev/multi-scrobbler@fe03bcd

+100 -44

src/backend/common/Cache.ts

··· 1 1 import { Cacheable, createKeyv, Keyv, KeyvStoreAdapter, KeyvOptions, CacheableOptions, KeyvCacheableMemory } from 'cacheable'; 2 2 import { FlatCache, FlatCacheOptions } from 'flat-cache'; 3 - import KeyvValkey from '@keyv/valkey'; 3 + import KeyvValkey, { KeyvValkeyOptions } from '@keyv/valkey'; 4 4 import dayjs, { Dayjs } from 'dayjs'; 5 5 import duration from 'dayjs/plugin/duration.js'; 6 6 import isBetween from 'dayjs/plugin/isBetween.js'; ··· 52 52 regexCache: ReturnType<typeof cacheFunctions>; 53 53 cacheTransform: Cacheable; 54 54 cacheClientScrobbles: Cacheable; 55 + cacheApi: Cacheable; 56 + hasInit: boolean = false; 55 57 56 58 logger: Logger; 57 59 ··· 66 68 67 69 const { 68 70 metadata: { 69 - provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? 'memory'), 71 + provider: mProvider = (process.env.CACHE_METADATA as (CacheMetadataProvider | undefined) ?? false), 70 72 connection: mConn = process.env.CACHE_METADATA_CONN, 71 73 ...restMetadata 72 74 } = {}, ··· 103 105 }; 104 106 105 107 this.regexCache = cacheFunctions(this.config.regex); 106 - this.cacheTransform = new Cacheable({primary: initMemoryCache({lruSize: 500})}); 107 - this.cacheTransform.stats.enabled = true; 108 - this.cacheClientScrobbles = new Cacheable({primary: initMemoryCache({lruSize: 100, ttl: '5m'})}); 109 - this.cacheClientScrobbles.stats.enabled = true; 108 + 109 + // for testing we default to in memory 110 + const inMemory = new Cacheable({primary: initMemoryCache({lruSize: 500, ttl: '1m'})}); 111 + this.cacheTransform = inMemory; 112 + this.cacheClientScrobbles = inMemory; 113 + this.cacheMetadata = inMemory; 114 + this.cacheAuth = inMemory; 115 + this.cacheScrobble = inMemory; 116 + this.cacheApi = inMemory; 110 117 } 111 118 112 119 init = async (enableCollectors: boolean = false) => { 113 120 await this.initMetadataCache(); 114 121 await this.initScrobbleCache(); 115 122 await this.initAuthCache(); 123 + 124 + this.cacheClientScrobbles = await this.initCacheable('Historical Scrobbles', {provider: 'memory', ttl: '2m', lruSize: 50}, {...this.config.metadata, ttl: '10m'}); 125 + this.cacheClientScrobbles.stats.enabled = true; 126 + this.cacheTransform = await this.initCacheable('Historical Scrobbles', {provider: 'memory', ttl: '2m', lruSize: 100}, {...this.config.metadata, ttl: '5m'}); 127 + this.cacheTransform.stats.enabled = true; 128 + this.cacheApi = await this.initCacheable('External API Responses', {provider: 'memory', ttl: '30s', lruSize: 100}, {...this.config.metadata, ttl: '5m'}); 129 + this.cacheApi.stats.enabled = true; 130 + 116 131 if(enableCollectors) { 117 132 this.enableCollectors(); 118 133 } ··· 124 139 { cache: this.cacheMetadata, name: 'metadata' }, 125 140 { cache: this.cacheScrobble, name: 'queued_scrobbles' }, 126 141 { cache: this.cacheTransform, name: 'transformer' }, 127 - { cache: this.cacheClientScrobbles, name: 'historical_scrobbles' } 142 + { cache: this.cacheClientScrobbles, name: 'historical_scrobbles' }, 143 + { cache: this.cacheApi, name: 'external_apis' } 128 144 ]; 129 145 130 146 this.cacheHits = new prom.Gauge({ ··· 199 215 // }); 200 216 } 201 217 202 - protected initCacheable = async (config: CacheConfig, cacheFor: string) => { 218 + protected initCacheable = async (cacheFor: string, primaryConfig: CacheConfig, secondaryConfig?: CacheConfig) => { 203 219 204 220 let logger = childLogger(this.logger, cacheFor); 205 - const providerHints = ['In-Memory (Primary)']; 206 - if(config.provider !== false) { 207 - providerHints.push(`${config.provider} (Secondary)`) 221 + const providerHints = []; 222 + if(primaryConfig.provider === false) { 223 + const cache = new Cacheable({primary: noopKeyv}); 224 + cache.stats.enabled = true; 225 + logger.verbose(`Cache Providers: Disabled`); 226 + return cache; 208 227 } 209 - logger.verbose(`Cache Providers: ${providerHints.join(' | ')}`) 228 + 229 + providerHints.push(`${primaryConfig.provider} (Primary)`) 230 + 231 + if(secondaryConfig === undefined || secondaryConfig.provider !== false) { 232 + providerHints.push(`Disabled (Secondary)`) 233 + } else { 234 + providerHints.push(`${secondaryConfig.provider} (Secondary)`); 235 + } 236 + logger.verbose(`Cache Providers: ${providerHints.join(' | ')}`); 210 237 211 238 const ns = `ms-${cacheFor.toLocaleLowerCase()}`; 212 239 213 240 const cacheOpts: CacheableOptions = { 214 - primary: initMemoryCache({ namespace: ns, lruSize: config.memory?.lruSize, ttl: config.memory?.ttl }) 241 + 215 242 } 216 243 217 - let secondaryCache: Keyv | KeyvStoreAdapter | undefined; 244 + try { 245 + cacheOpts.primary = await this.initCachableType(ns, primaryConfig, logger); 246 + } catch (e) { 247 + throw new Error('Could not init primary cache', {cause: e}); 248 + } 218 249 219 - if (config.provider === 'valkey') { 220 - logger.debug(`Building valkey cache from ${config.connection}`); 250 + if(secondaryConfig !== undefined && secondaryConfig.provider !== false) { 221 251 try { 222 - secondaryCache = await initValkeyCache(ns, config.connection); 223 - logger.debug('valkey cache connected'); 252 + cacheOpts.secondary = await this.initCachableType(ns, secondaryConfig, logger); 224 253 } catch (e) { 225 254 this.logger.warn(e); 226 255 } 227 - } else if (config.provider === 'file') { 228 - logger.debug(`Building file cache from ${path.join(config.connection, `${ns}.cache`)}`); 256 + } 257 + 258 + const cache = new Cacheable(cacheOpts); 259 + cache.stats.enabled = true; 260 + return cache; 229 261 262 + } 263 + 264 + protected initCachableType = async (namespace: string, config: CacheConfig, logger: Logger): Promise<Keyv<any> | KeyvStoreAdapter> => { 265 + 266 + if (config.provider === 'memory') { 267 + return initMemoryCache({ namespace, lruSize: config.lruSize, ttl: config.ttl }); 268 + } 269 + 270 + if (config.provider === 'valkey') { 271 + logger.debug(`Building valkey cache from ${config.connection}`); 230 272 try { 231 - const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${ns}.cache` }, logger); 232 - secondaryCache = keyvFile; 273 + const cache = await initValkeyCache(namespace, config.connection, undefined, {ttl: config.ttl}); 274 + logger.debug('valkey cache connected'); 275 + return cache; 233 276 } catch (e) { 234 - logger.warn(e); 277 + throw e; 235 278 } 236 279 } 280 + if (config.provider === 'file') { 281 + logger.debug(`Building file cache from ${path.join(config.connection, `${namespace}.cache`)}`); 237 282 238 - if(secondaryCache !== undefined) { 239 - cacheOpts.secondary = secondaryCache; 283 + try { 284 + const [keyvFile] = await initFileCache({ ...config, cacheDir: config.connection, cacheId: `${namespace}.cache` }, {ttl: config.ttl}, logger); 285 + return keyvFile; 286 + } catch (e) { 287 + throw e; 288 + } 240 289 } 241 - const cache = new Cacheable(cacheOpts); 242 - cache.stats.enabled = true; 243 - return cache; 244 - 245 290 } 246 291 247 292 initScrobbleCache = async () => { 248 - if (this.cacheScrobble === undefined) { 293 + if (!this.hasInit) { 249 294 if (!asCacheScrobbleProvider(this.config.scrobble.provider)) { 250 295 throw new Error(`Cache Scrobble provider '${this.config.scrobble.provider}' must be one of: memory, valkey, file`); 251 296 } 252 297 253 - this.cacheScrobble = await this.initCacheable(this.config.scrobble, 'Scrobble'); 298 + this.cacheScrobble = await this.initCacheable('Scrobble', this.config.scrobble); 254 299 } 255 300 } 256 301 257 302 initMetadataCache = async () => { 258 - if (this.cacheMetadata === undefined) { 303 + if (!this.hasInit) { 259 304 if (!asCacheMetadataProvider(this.config.metadata.provider)) { 260 305 throw new Error(`Cache Metadata provider '${this.config.metadata.provider}' must be one of: memory, valkey`); 261 306 } 262 307 263 - this.cacheMetadata = await this.initCacheable(this.config.metadata, 'Metadata'); 308 + this.cacheMetadata = await this.initCacheable('Metadata', {provider: 'memory', lruSize: 100, ttl: '3m'}, {...this.config.metadata, ttl: '15m'}); 264 309 } 265 310 } 266 311 267 312 initAuthCache = async () => { 268 - if (this.cacheAuth === undefined) { 313 + if (!this.hasInit) { 269 314 if (!asCacheAuthProvider(this.config.auth.provider)) { 270 315 throw new Error(`Cache Auth provider '${this.config.auth.provider}' must be one of: memory, valkey, file`); 271 316 } 272 317 273 - this.cacheAuth = await this.initCacheable(this.config.auth, 'Auth'); 318 + this.cacheAuth = await this.initCacheable('Auth', {provider: 'memory', ttl: '3m'}, this.config.auth); 274 319 } 275 320 } 276 321 ··· 279 324 280 325 export const initMemoryCache = <T = any>(opts: Parameters<typeof createKeyv>[0] = {}): Keyv<T> | KeyvStoreAdapter => { 281 326 const { 282 - ttl = '1h', 327 + ttl = '60s', 283 328 lruSize = 200, 284 329 ...restOpts 285 330 } = opts; ··· 359 404 } 360 405 } 361 406 362 - export const initFileCache = async (opts: FlatCacheOptions = {}, logger: Logger = loggerNoop): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => { 407 + export const initFileCache = async (opts: FlatCacheOptions = {}, keyvOpts: KeyvOptions = {}, logger: Logger = loggerNoop): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => { 363 408 const flatCache = flatCacheCreate(opts); 364 409 try { 365 410 await flatCacheLoad(flatCache, logger); ··· 373 418 const cache = new Keyv({ 374 419 store: flatCache, 375 420 throwOnErrors: true, 376 - ...typesonMarshalling 421 + ...typesonMarshalling, 422 + ...keyvOpts 377 423 }); 378 424 cache.stats.enabled = true; 379 425 return [cache, flatCache]; ··· 382 428 } 383 429 } 384 430 385 - export const valkeyCacheCreate = (ns: string, ...args: ConstructorParameters<typeof KeyvValkey>): Keyv => { 386 - const [connection, valkeyOpts = {}] = args; 431 + export const valkeyCacheCreate = (ns: string, connection: string, valkeyOpts: KeyvValkeyOptions = {}, keyvOpts: KeyvOptions = {}): Keyv => { 387 432 const valkey = new KeyvValkey(connection, { maxRetriesPerRequest: 5, connectTimeout: 1100, ...valkeyOpts }); 388 433 const kv = new Keyv({ 389 434 store: valkey, 390 435 throwOnErrors: true, 391 436 namespace: ns, 392 - ...typesonMarshalling 437 + ...typesonMarshalling, 438 + ...keyvOpts 393 439 }); 394 440 kv.stats.enabled = true; 395 441 return kv; 396 442 } 397 443 398 - export const initValkeyCache = async (ns: string, ...args: ConstructorParameters<typeof KeyvValkey>): Promise<Keyv> => { 399 - const kv = valkeyCacheCreate(ns, ...args); 444 + export const initValkeyCache = async (ns: string, connection: string, valkeyOpts: KeyvValkeyOptions = {}, keyvOpts: KeyvOptions = {}): Promise<Keyv> => { 445 + const kv = valkeyCacheCreate(ns, connection, valkeyOpts, keyvOpts); 400 446 try { 401 447 await kv.get('test'); 402 448 return kv; 403 449 } catch (e) { 404 - throw new Error(`Unable to connect to cache ${args[0]}`, { cause: e }) 450 + throw new Error(`Unable to connect to cache ${connection}`, { cause: e }) 405 451 } 406 452 } 407 453 ··· 426 472 secondary = cache.secondary.stats[statName]; 427 473 } 428 474 return [primary, secondary]; 475 + } 476 + 477 + const noopKeyv: KeyvStoreAdapter = { 478 + opts: {}, 479 + namespace: 'noop', 480 + get: (_) => undefined, 481 + set: (_, __, ___) => undefined, 482 + delete: (_) => undefined, 483 + clear: () => Promise.resolve(), 484 + on: (_, __) => undefined 429 485 }

+1 -1

src/backend/common/MaybeLogger.ts

··· 48 48 } 49 49 } 50 50 } 51 - export const noopLog = (_: any, ...rest: any) => undefined; 51 + const noopLog = (_: any, ...rest: any) => undefined; 52 52 53 53 export const loggerNoop: Logger = { 54 54 trace: noopLog,

+16 -3

src/backend/common/infrastructure/Atomic.ts

··· 230 230 connection?: string; 231 231 [key: string]: any 232 232 } 233 + export interface CacheMemoryConfig extends CacheConfig<'memory'> { 234 + lruSize?: number 235 + ttl?: number 236 + } 237 + export interface CacheValkeyConfig extends CacheConfig<'valkey'> { 238 + } 239 + export interface CacheFileConfig extends CacheConfig<'file'> { 240 + } 241 + export interface CacheNoopConfig extends CacheConfig<false> { 242 + } 243 + 244 + export type CacheConfigType = CacheMemoryConfig | CacheValkeyConfig | CacheFileConfig | CacheNoopConfig; 245 + 233 246 export type CacheMetadataProvider = CacheProvider;//Exclude<CacheProvider, 'file'>; 234 - export type CacheMetadataConfig = CacheConfig<CacheMetadataProvider>; 247 + export type CacheMetadataConfig = CacheConfigType; 235 248 export const asCacheProvider = (val: boolean | string): val is CacheProvider => { 236 249 if(typeof val === 'string') { 237 250 return ['memory', 'valkey', 'file'].includes(val); ··· 240 253 } 241 254 export const asCacheMetadataProvider = (val: any): val is CacheScrobbleProvider => asCacheProvider(val); 242 255 export type CacheScrobbleProvider = CacheProvider; 243 - export type CacheScrobbleConfig = CacheConfig<CacheScrobbleProvider>; 256 + export type CacheScrobbleConfig = CacheConfigType; 244 257 export const asCacheScrobbleProvider = (val: any): val is CacheScrobbleProvider => asCacheProvider(val); 245 258 246 259 export type CacheAuthProvider = CacheProvider; 247 - export type CacheAuthConfig = CacheConfig<CacheAuthProvider>; 260 + export type CacheAuthConfig = CacheConfigType; 248 261 export const asCacheAuthProvider = (val: any): val is CacheAuthProvider => asCacheProvider(val); 249 262 export interface CacheConfigOptions { 250 263 metadata?: CacheMetadataConfig;

+6 -2

src/backend/tests/scrobbler/scrobblers.test.ts

··· 22 22 import { DEFAULT_CONSOLIDATE_DURATION, DEFAULT_GROUP_DURATION, groupPlaysToTimeRanges } from '../../utils/ListenFetchUtils.js'; 23 23 import { asPlay } from '../../../core/tests/utils/fixtures.js'; 24 24 import { nanoid } from 'nanoid'; 25 + import { getRoot } from '../../ioc.js'; 26 + import { transientCache } from '../utils/CacheTestUtils.js'; 25 27 26 28 chai.use(asPromised); 27 29 ··· 511 513 512 514 afterEach(function () { 513 515 MockDate.reset(); 516 + const root = getRoot(); 517 + root.upsert({ cache: () => transientCache }); 514 518 }); 515 519 516 520 it('Calls timerange func to get SOT scrobbles when none exists', async function() { ··· 547 551 scrobbler.startScrobbling().then(() => null); 548 552 await emptied; 549 553 scrobbler.tryStopScrobbling().then(() => null); 550 - expect(sp.calledOnce).is.true; 554 + expect(sp.callCount).to.eq(1); 551 555 }); 552 556 553 557 it('Uses separate timerange calls when scrobbles are not closely grouped', async function() { ··· 567 571 scrobbler.startScrobbling().then(() => null); 568 572 await emptied; 569 573 scrobbler.tryStopScrobbling().then(() => null); 570 - expect(sp.calledTwice).is.true; 574 + expect(sp.callCount).to.eq(2); 571 575 }); 572 576 573 577 it('Gets fresh timerange if TTL of staleAfter has passed', async function() {

+1 -1

src/backend/tests/utils/CacheTestUtils.ts

··· 1 1 import { loggerTest } from "@foxxmd/logging"; 2 2 import { MSCache } from "../../common/Cache.js"; 3 3 4 - export const transientCache = () => new MSCache(loggerTest, {scrobble: {provider: 'memory'}, auth: {provider: 'memory'}}); 4 + export const transientCache = () => new MSCache(loggerTest, {scrobble: {provider: 'memory'}, auth: {provider: 'memory'}, metadata: {provider: 'memory'}});

Configure Feed

Configure Feed