feat(database): Add id awareness to plays and implement basic api GET for sources · foxxmd.dev/multi-scrobbler@3ef4854

+14

src/backend/utils.ts

··· 225 225 return newObj; 226 226 } 227 227 228 + export const removeEmptyArrays = <T extends Record<string, any>>(obj: T): T => { 229 + const newObj: any = {}; 230 + Object.keys(obj).forEach((key) => { 231 + if(Array.isArray(obj[key])) { 232 + if(obj[key].length !== 0) { 233 + newObj[key] = obj[key]; 234 + } 235 + } else { 236 + newObj[key] = obj[key]; 237 + } 238 + }); 239 + return newObj; 240 + } 241 + 228 242 export const remoteHostIdentifiers = (req: Request): RemoteIdentityParts => { 229 243 const remote = req.connection.remoteAddress; 230 244 const proxyRemote = Array.isArray(req.headers["x-forwarded-for"]) ? req.headers["x-forwarded-for"][0] : req.headers["x-forwarded-for"];

+3

src/core/Atomic.ts

··· 202 202 203 203 seenAt?: D 204 204 205 + dbUid?: string 206 + dbId?: number 207 + 205 208 /* 206 209 * If applicable, the name of the Service providing the track (Spotify, Tidal, etc...) 207 210 */

+7 -1

src/backend/server/api.ts

··· 262 262 // @ts-expect-error TS(2339): Property 'scrobbleSource' does not exist on type '... Remove this comment to see the full error message 263 263 scrobbleSource: source, 264 264 query: { 265 - upstream = 'false' 265 + upstream = 'false', 266 + next: queryNext = 'false', 267 + ...rest 266 268 } 267 269 } = req; 268 270 ··· 278 280 return res.status(500).json({message: e.message}); 279 281 } 280 282 } else { 283 + if(queryNext === 'true') { 284 + return res.json(await (source as AbstractSource).getRecentPlaysApi(rest)); 285 + } 281 286 result = await (source as AbstractSource).getFlatRecentlyDiscoveredPlays(); 287 + 282 288 } 283 289 } 284 290

+19 -4

src/backend/sources/AbstractSource.ts

··· 46 46 import { normalizeStr } from '../utils/StringUtils.js'; 47 47 import { spawn, catchAbortError, isAbortError, rethrowAbortError, delay, forever, AbortError, throwIfAborted } from 'abort-controller-x'; 48 48 import { AbortedError, generateLoggableAbortReason } from '../common/errors/MSErrors.js'; 49 - import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts } from '../common/database/drizzle/repositories/PlayRepository.js'; 49 + import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, queryArgsFromRequest, QueryPlaysOpts, RequestPlayQuery } from '../common/database/drizzle/repositories/PlayRepository.js'; 50 50 import { asPlay } from '../../core/PlayMarshalUtils.js'; 51 51 52 52 export interface RecentlyPlayedOptions { ··· 212 212 // TODO make this more descriptive? or move it elsewhere 213 213 recentlyPlayedTrackIsValid = (playObj: PlayObject) => true 214 214 215 - protected addPlayToDiscovered = async (play: PlayObject) => { 215 + protected addPlayToDiscovered = async (play: PlayObject): Promise<PlayObject> => { 216 216 const platformId = this.multiPlatform ? genGroupId(play) : SINGLE_USER_PLATFORM_ID; 217 217 const playRow = await this.playRepo.createPlays([(playToRepositoryCreatePlayOpts({play, componentId: this.dbComponent.id, state: 'discovered'}))]); 218 218 const recentPlays = await this.getRecentlyDiscoveredPlaysByPlatform(platformId, false); ··· 231 231 this.logger.info(`Discovered => ${buildTrackString(play)}`); 232 232 this.emitEvent('discovered', {play}); 233 233 this.discoveredCounter.labels(this.getPrometheusLabels()).inc(); 234 + play.meta.dbId = playRow[0].id; 235 + return play; 234 236 } 235 237 236 238 getFlatRecentlyDiscoveredPlays = async (): Promise<PlayObject[]> => { ··· 241 243 } 242 244 return list.flat().sort(sortByNewestPlayDate); 243 245 //Array.from(this.recentDiscoveredPlays.values()).map(x => x.data).flat(3).sort(sortByNewestPlayDate) 246 + } 247 + 248 + getRecentPlaysApi = async (query: RequestPlayQuery) => { 249 + const res = await this.playRepo.findPlays({ 250 + componentId: this.dbComponent.id, 251 + limit: 100, 252 + ...queryArgsFromRequest(query) 253 + }); 254 + return res.map((x) => { 255 + const {id, ...rest} = x; 256 + return rest; 257 + }) 244 258 } 245 259 246 260 protected recentDiscoveredCacheKey = (platformId: PlayPlatformId | string) => { ··· 330 344 options.signal?.throwIfAborted(); 331 345 if(!(await this.alreadyDiscovered(play, options))) { 332 346 options.signal?.throwIfAborted() 333 - await this.addPlayToDiscovered(play); 334 - newDiscoveredPlays.push(play); 347 + const hydratedPlay = await this.addPlayToDiscovered(play); 348 + newDiscoveredPlays.push(hydratedPlay); 335 349 } 336 350 } 337 351 if(newDiscoveredPlays.length > 0) { ··· 362 376 363 377 if(newDiscoveredPlays.length > 0) { 364 378 if(!this.shouldScrobble(options.discoverLocation)) { 379 + await this.playRepo.setStateById('discarded', newDiscoveredPlays.map(x => x.meta.dbId)); 365 380 return; 366 381 } 367 382 newDiscoveredPlays.sort(sortByOldestPlayDate);

+6 -3

src/backend/common/database/drizzle/drizzleTypes.ts

··· 1 - import { DBQueryConfig, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals } from "drizzle-orm"; 1 + import { DBQueryConfig, DBQueryConfigWith, ExtractTablesFromSchema, KnownKeysOnly, RelationFieldsFilterInternals } from "drizzle-orm"; 2 2 import { components, playInputs, plays, queueStates, relations } from "./schema/schema.js"; 3 3 import {TSchema, TableName, Schema } from "./schema/schema.js"; 4 4 ··· 23 23 // https://github.com/drizzle-team/drizzle-orm/issues/695 most examples 24 24 // https://github.com/drizzle-team/drizzle-orm/discussions/2316 relation focused 25 25 // https://github.com/drizzle-team/drizzle-orm/issues/1319 26 + 27 + //type p = TSchema['plays']['relations']; 28 + export type FindWith<T extends TableName> = DBQueryConfigWith<TSchema, TSchema[T]['relations']>; 26 29 export type QueryConfig<T extends TableName> = DBQueryConfig<"many", TSchema, TSchema[T]>; 27 - export type FindMany<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> 28 - export type FindOne<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> 30 + export type FindMany<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"many", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>} 31 + export type FindOne<T extends TableName> = Pick<KnownKeysOnly<QueryConfig<T>, DBQueryConfig<"one", TSchema, TSchema[T]>>, 'where' | 'orderBy' | 'limit' | 'offset' | 'extras'> & {with?: FindWith<T>} 29 32 export type FindWhere<T extends TableName> = QueryConfig<T>['where']; 30 33 31 34 export type CompareOp<T> = Pick<RelationFieldsFilterInternals<T>, 'gt' | 'gte' | 'eq' | 'lt' | 'lte' | 'ne'>

+22 -1

src/backend/common/database/drizzle/entityUtils.ts

··· 1 1 import assert from "node:assert"; 2 - import { PlayNew } from "./drizzleTypes.js"; 2 + import { PlayNew, PlaySelect } from "./drizzleTypes.js"; 3 3 import { PlayInputNew } from "./drizzleTypes.js"; 4 4 import { QueueStateNew } from "./drizzleTypes.js"; 5 5 import { ComponentNew } from "./drizzleTypes.js"; 6 6 import { MarkOptional } from "ts-essentials"; 7 7 import { ErrorLike, PlayObject } from "../../../../core/Atomic.js"; 8 8 import dayjs, { Dayjs } from "dayjs"; 9 + import { asPlay } from "../../../../core/PlayMarshalUtils.js"; 9 10 10 11 export const generateComponentEntity = (data: MarkOptional<ComponentNew, 'uid'>): ComponentNew => { 11 12 assert(data.name !== undefined, 'Must provide name'); ··· 31 32 seenAt, 32 33 ...restOpts 33 34 } 35 + } 36 + 37 + export type PlayHydateOptions = 'asPlay' | 'id' | 'uid'; 38 + 39 + export const hydratePlaySelect = (select: PlaySelect, opts: PlayHydateOptions[]): PlayObject => { 40 + if(opts.length === 0) { 41 + return select.play; 42 + } 43 + 44 + let res = select.play; 45 + if(opts.includes('asPlay')) { 46 + res = asPlay(res); 47 + } 48 + if(opts.includes('uid')) { 49 + res.meta.dbUid = select.uid; 50 + } 51 + if(opts.includes('id')) { 52 + res.meta.dbId = select.id; 53 + } 54 + return res; 34 55 } 35 56 36 57 export const generateInputEntity = (data: PlayInputNew): PlayInputNew => {

+101 -6

src/backend/common/database/drizzle/repositories/PlayRepository.ts

··· 2 2 import { DbConcrete, getDb, runTransaction } from "../drizzleUtils.js"; 3 3 import { loggerNoop } from "../../../MaybeLogger.js"; 4 4 import { PlayObject } from "../../../../../core/Atomic.js"; 5 - import { generateInputEntity, generatePlayEntity, PlayEntityOpts } from "../entityUtils.js"; 5 + import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js"; 6 6 import { playInputs, plays, relations } from "../schema/schema.js"; 7 7 import { PlayNew, PlaySelect, PlayInputNew, FindWhere, FindMany, CompareOpKey } from "../drizzleTypes.js";; 8 8 import { MarkOptional, MarkRequired, PathValue } from "ts-essentials"; 9 - import { genGroupIdStrFromPlay, removeUndefinedKeys } from "../../../../utils.js"; 9 + import { genGroupIdStrFromPlay, removeEmptyArrays, removeUndefinedKeys } from "../../../../utils.js"; 10 10 import dayjs, { Dayjs } from "dayjs"; 11 11 import { RelationsFieldFilter, eq, inArray, ne, notInArray, desc, asc, and } from "drizzle-orm"; 12 12 import { CompactableProperty, RetentionOptions, retentionPlayTypes } from "../../../infrastructure/config/database.js"; 13 13 import { shortTodayAwareFormat } from "../../../../../core/TimeUtils.js"; 14 14 import { buildDateCompare, CompareDateOp, DrizzleBaseRepository } from "./BaseRepository.js"; 15 + import { asPlay } from "../../../../../core/PlayMarshalUtils.js"; 16 + import assert from "node:assert"; 17 + import { parseArrayFromMaybeString } from "../../../../utils/StringUtils.js"; 15 18 16 19 // https://github.com/drizzle-team/drizzle-orm/issues/695 may be useful for typing models with relations? 17 20 ··· 25 28 platformId?: string 26 29 seenAt?: CompareDateOp 27 30 playedAt?: CompareDateOp 31 + uid?: string[] 28 32 } 29 33 34 + export type WithPlayRelation = 'input' | 'parent' | 'parent-input'; 30 35 export interface QueryPlaysOpts extends PlayWhereOpts { 31 36 sort?: 'seenAt' | 'playedAt' 32 37 order?: 'asc' | 'desc' 38 + with?: WithPlayRelation[] 33 39 limit?: number 34 40 offset?: number 35 41 } ··· 45 51 super(db, 'plays', 'Plays', opts); 46 52 } 47 53 48 - createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[]) => { 54 + createPlays = async (entitiesOpts: RepositoryCreatePlayOpts[], opts: {hydrate?: PlayHydateOptions[]} = {}) => { 49 55 56 + const { 57 + hydrate = [] 58 + } = opts; 50 59 let playRows: PlaySelect[]; 51 60 52 61 await runTransaction(this.db, async () => { ··· 79 88 80 89 }); 81 90 82 - return playRows; 91 + return playRows.map(x => ({...x, play: hydratePlaySelect(x, hydrate)})); 83 92 } 84 93 85 - findPlays = async (args: QueryPlaysOpts): Promise<PlaySelect[]> => { 86 - //let oldQuery: Parameters<typeof this.db.query.plays.findMany>[0] = {}; 94 + findPlays = async (args: QueryPlaysOpts, opts: {hydrate?: PlayHydateOptions[]} = {}): Promise<PlaySelect[]> => { 95 + const { 96 + hydrate = [] 97 + } = opts; 98 + // this does not work as type for query variable 99 + // it erases the result type for some reason 100 + // 101 + // Parameters<typeof this.db.query.plays.findMany>[0] 102 + 103 + // this does work but it is also integrated into FindWith 104 + //let withQuery: Parameters<typeof this.db.query.plays.findMany>[0]['with'] = undefined; 105 + 87 106 let query: FindMany<'plays'> = { 88 107 limit: args.limit, 89 108 offset: args.offset ··· 100 119 id: 'asc' 101 120 } 102 121 } 122 + 123 + if(args.with !== undefined) { 124 + query.with = {}; 125 + for(const w of args.with) { 126 + switch (w) { 127 + case 'input': 128 + query.with.input = true; 129 + break; 130 + case 'parent': 131 + query.with.parent = true; 132 + break; 133 + case 'parent-input': 134 + query.with.parent = { 135 + with: { 136 + input: true 137 + } 138 + }; 139 + break; 140 + default: 141 + throw new Error(`Unknown relation ${w}`); 142 + } 143 + } 144 + } 103 145 query = removeUndefinedKeys(query); 104 146 const results = await this.db.query.plays.findMany(query); 147 + if(hydrate.length > 0) { 148 + return results.map((x) => ({...x, play: hydratePlaySelect(x, hydrate)})); 149 + } 105 150 return results; 151 + } 152 + 153 + setStateById = async (state: PlayNew['state'], ids: number[]): Promise<void> => { 154 + const validIds = ids.filter(x => x !== undefined && x !== null); 155 + assert(validIds.length > 0, `Should not pass empty array of ids, after filtering, to update state. Original ids list: ${ids}`); 156 + await this.db.update(plays).set({state}).where(inArray(plays.id, ids)); 106 157 } 107 158 108 159 deletePlays = async (playsData: (Pick<PlaySelect, 'id'> | number)[]) => { ··· 319 370 if(args.platformId !== undefined) { 320 371 where.platformId = args.platformId 321 372 } 373 + if(args.uid !== undefined) { 374 + where.uid = { 375 + in: args.uid 376 + } 377 + } 322 378 return where; 323 379 } 324 380 ··· 356 412 }, 357 413 platformId: genGroupIdStrFromPlay(data.play) 358 414 } 415 + } 416 + 417 + export type RequestPlayQuery = Partial< Record<keyof Exclude<QueryPlaysOpts, 'componentId' | 'platformId'>, string>>; 418 + 419 + export const queryArgsFromRequest = (rec: RequestPlayQuery): QueryPlaysOpts => { 420 + 421 + const { 422 + state, 423 + stateNot, 424 + uid, 425 + with: withQuery, 426 + seenAt, 427 + playedAt, 428 + limit, 429 + sort, 430 + order, 431 + offset, 432 + componentId, 433 + ...rest 434 + } = rec; 435 + 436 + let queryArgs: QueryPlaysOpts = removeEmptyArrays<QueryPlaysOpts>({ 437 + state: parseArrayFromMaybeString(state) as PlaySelect['state'][], 438 + stateNot: parseArrayFromMaybeString(stateNot) as PlaySelect['state'][], 439 + uid: parseArrayFromMaybeString(uid), 440 + with: parseArrayFromMaybeString(withQuery) as WithPlayRelation[], 441 + sort: sort as 'playedAt' | 'seenAt', 442 + order: order as 'asc' | 'desc', 443 + ...rest 444 + }); 445 + 446 + if(limit !== undefined) { 447 + queryArgs.limit = Number.parseInt(limit); 448 + } 449 + if(offset !== undefined) { 450 + queryArgs.offset = Number.parseInt(offset); 451 + } 452 + 453 + return queryArgs; 359 454 }

Configure Feed

Configure Feed