feat: More artist parsing improvements · foxxmd.dev/multi-scrobbler@a3d8372

+48 -1

src/backend/tests/plays/playParsing.test.ts

··· 4 4 import { after, before, describe, it } from 'mocha'; 5 5 6 6 import { asPlays, generateArtistsStr, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; 7 - import { parseArtistCredits, parseCredits } from "../../utils/StringUtils.js"; 7 + import { parseArtistCredits, parseContextAwareStringList, parseCredits } from "../../utils/StringUtils.js"; 8 8 9 9 describe('Parsing Artists from String', function() { 10 10 ··· 18 18 '${str}' 19 19 Expected => ${allArtists.join(' || ')} 20 20 Found => ${parsed.join(' || ')}`) 21 + 21 22 .eql(parsed) 23 + } 24 + }); 25 + 26 + it('Parses & as "local" joiner when other delimiters present', function () { 27 + 28 + const data = [{ 29 + str: `Melendi \\ Ryan Lewis \\ The Righteous Brothers (featuring Joan Jett & The Blackhearts \\ Robin Schulz)`, 30 + expected: ['Melendi', 'Ryan Lewis', 'The Righteous Brothers', 'Joan Jett & The Blackhearts', 'Robin Schulz'] 31 + }, { 32 + str: `Gigi D'Agostino \\ YOASOBI (vs Sam Hunt, Lisa Loeb & Booba)`, 33 + expected: [`Gigi D'Agostino`, 'YOASOBI', 'Sam Hunt', 'Lisa Loeb', 'Booba'] 34 + }]; 35 + 36 + for(const d of data) { 37 + const credits = parseArtistCredits(d.str); 38 + const parsed = [credits.primary].concat(credits.secondary ?? []) 39 + expect(d.expected).eql(parsed) 40 + } 41 + 42 + }); 43 + 44 + it('Only parses & as "global" joiner when no other delimiters present', function () { 45 + 46 + const data = [{ 47 + str: `Melendi & Ryan Lewis & The Righteous Brothers (featuring The Blackhearts \\ Robin Schulz)`, 48 + expected: ['Melendi', 'Ryan Lewis', 'The Righteous Brothers', 'The Blackhearts', 'Robin Schulz'] 49 + }]; 50 + 51 + for(const d of data) { 52 + const credits = parseArtistCredits(d.str); 53 + const parsed = [credits.primary].concat(credits.secondary ?? []) 54 + expect(d.expected).eql(parsed) 55 + } 56 + }); 57 + 58 + it('Parses secondary free regex', function () { 59 + 60 + const data = [{ 61 + str: `Diddy & Grand Funk Railroad feat. Daya & (G)I-DLE`, 62 + expected: ['Diddy', 'Grand Funk Railroad', 'Daya', '(G)I-DLE'] 63 + }]; 64 + 65 + for(const d of data) { 66 + const credits = parseArtistCredits(d.str); 67 + const parsed = [credits.primary].concat(credits.secondary ?? []) 68 + expect(d.expected).eql(parsed) 22 69 } 23 70 }); 24 71

+31 -7

src/backend/tests/utils/PlayTestUtils.ts

··· 9 9 import { sortByNewestPlayDate } from "../../utils.js"; 10 10 import { NO_DEVICE, NO_USER, PlayerStateDataMaybePlay, PlayPlatformId, ReportedPlayerStatus } from '../../common/infrastructure/Atomic.js'; 11 11 import { arrayListAnd } from '../../../core/StringUtils.js'; 12 + import { findDelimiters } from '../../utils/StringUtils.js'; 12 13 13 14 dayjs.extend(utc) 14 15 dayjs.extend(isBetween); ··· 180 181 181 182 export const generateArtist = () => faker.music.artist; 182 183 183 - export const generateArtists = (num?: number, max: number = 3) => { 184 - // if(num !== undefined) { 185 - // return Array(num).map(x => faker.music.artist); 186 - // } 184 + export const generateArtists = (num?: number, max: number = 3, opts: {ambiguousJoinedNames?: boolean, trailingAmpersand?: boolean} = {}) => { 187 185 if(num === 0 || max === 0) { 188 186 return []; 189 187 } 190 - return faker.helpers.multiple(faker.music.artist, {count: {min: num ?? 1, max: num ?? max}}); 188 + let artists = faker.helpers.multiple(faker.music.artist, {count: {min: num ?? 1, max: num ?? max}}); 189 + 190 + const { 191 + trailingAmpersand = false, 192 + ambiguousJoinedNames = false 193 + } = opts; 194 + 195 + if(!trailingAmpersand) { 196 + // its really hard to parse an artist name that contains an '&' when it comes at the end of a list 197 + // because its ambigious if the list is joining the list with & or if & is part of the artist name 198 + // so by default don't generate these (we test for specific scenarios in playParsing.test.ts) 199 + while(artists[artists.length - 1].includes('&')) { 200 + artists = artists.slice(0, artists.length - 1).concat(faker.music.artist()); 201 + } 202 + } 203 + if(!ambiguousJoinedNames) { 204 + artists = artists.map(x => { 205 + let a = x; 206 + let foundDelims = findDelimiters(a); 207 + while(foundDelims !== undefined && foundDelims.length > 0 && !(foundDelims.length === 1 && foundDelims[0] === '&')) { 208 + a = faker.music.artist(); 209 + foundDelims = findDelimiters(a); 210 + } 211 + return a; 212 + }); 213 + } 214 + return artists; 191 215 } 192 216 193 217 export interface ArtistGenerateOptions { ··· 223 247 let finalJoinerPrimary: string = joinerPrimary; 224 248 if(primaryOpts.finalJoiner !== false) { 225 249 if(primaryOpts.finalJoiner === undefined) { 226 - if(joinerPrimary === ',') { 250 + if(joinerPrimary === ',' && !primaryArt.some(x => x.includes('&'))) { 227 251 finalJoinerPrimary = faker.helpers.arrayElement(JOINERS_FINAL); 228 252 } 229 253 ··· 242 266 let finalJoinerSecondary: string = joinerSecondary; 243 267 if(secondaryOpts.finalJoiner !== false) { 244 268 if(secondaryOpts.finalJoiner === undefined) { 245 - if(joinerSecondary === ',') { 269 + if(joinerSecondary === ',' && !secondaryArt.some(x => x.includes('&'))) { 246 270 finalJoinerSecondary = faker.helpers.arrayElement(JOINERS_FINAL); 247 271 } 248 272 } else {

+48 -4

src/backend/utils/StringUtils.ts

··· 1 1 import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness"; 2 2 import { PlayObject } from "../../core/Atomic.js"; 3 3 import { asPlayerStateData, DELIMITERS, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; 4 - import { genGroupIdStr, getPlatformIdFromData, parseRegexSingleOrFail } from "../utils.js"; 4 + import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; 5 5 import { buildTrackString } from "../../core/StringUtils.js"; 6 6 7 7 const {levenStrategy, diceStrategy} = strategies; ··· 61 61 * !!!! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ******* 62 62 * 63 63 * */ 64 - export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?<joiner>ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?<credits>(?:.+?(?= - |\s*[([]))|(?:.*))(?<creditsSuffix>.*)/i); 64 + export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?<joiner>ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?<credits>(?:.+?(?= - |\s*[([].+[)\]]$))|(?:.*))(?<creditsSuffix>.*)/i); 65 65 66 66 const SECONDARY_REGEX_STRATS: RegExp[] = [SECONDARY_CAPTURED_REGEX, SECONDARY_FREE_REGEX]; 67 67 ··· 116 116 for(const strat of SECONDARY_REGEX_STRATS) { 117 117 const secCredits = parseRegexSingleOrFail(strat, results.named.secondary); 118 118 if(secCredits !== undefined) { 119 - secondary = parseStringList(secCredits.named.credits as string, delims) 119 + secondary = parseContextAwareStringList(secCredits.named.credits as string, delims) 120 120 suffix = secCredits.named.creditsSuffix; 121 121 break; 122 122 } ··· 148 148 if (withJoiner !== undefined) { 149 149 // all this does is make sure and "ft" or parenthesis/brackets are separated -- 150 150 // it doesn't also separate primary artists so do that now 151 - const primaries = parseStringList(withJoiner.primary, delims); 151 + const primaries = parseContextAwareStringList(withJoiner.primary, delims); 152 152 if (primaries.length > 1) { 153 153 return { 154 154 primary: primaries[0], ··· 181 181 const explodedStrings = acc.map(x => x.split(curr)); 182 182 return explodedStrings.flat(1); 183 183 }, [str]).map(x => x.trim()); 184 + } 185 + export const parseContextAwareStringList = (str: string, delimiters: string[] = [',', '/', '\\'], opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { 186 + if (delimiters.length === 0) { 187 + return [str]; 188 + } 189 + // bypass tokens using slashes without spaces 190 + const cleanStr = bypassJoiners(str); 191 + const nonAmpersandDelims = delimiters.some(x => cleanStr.includes(x)); 192 + const shouldIgnoreGlobalAmpersand = opts.ignoreGlobalAmpersand ?? nonAmpersandDelims; 193 + 194 + let awareList: string[] = []; 195 + 196 + const list = parseStringList(cleanStr, nonAmpersandDelims === false && shouldIgnoreGlobalAmpersand === false ? ['&'] : delimiters); 197 + if(shouldIgnoreGlobalAmpersand && list.length > 1 && list[list.length - 1].includes('&') && nonAmpersandDelims) { //&& !list[list.length - 1].includes('& the') 198 + awareList = list.slice(0, list.length - 1).concat(list[list.length - 1].split('&') ); 199 + } else { 200 + awareList = list; 201 + } 202 + return awareList.map(x =>rejoinBypassed(x.trim())); 203 + } 204 + 205 + const bypassJoinerMap = [ 206 + { 207 + rejoin: str => str.replaceAll(/(.*?\S)(\^\^\^)(\S.*?)/g, '$1/$3'), 208 + bypass: str => str.replaceAll(/(.*?\S)(\/)(\S.*?)/g, '$1^^^$3') 209 + }, 210 + { 211 + rejoin: str => str.replaceAll(/(.*)(###)(.*)/g, '$1\\$3'), 212 + bypass: str => str.replaceAll(/(.*\S)(\\)(.*\S)/g, '$1###$3') 213 + } 214 + ]; 215 + export const bypassJoiners = (str: string): string => { 216 + let bypassed: string = str; 217 + for(const b of bypassJoinerMap) { 218 + bypassed = b.bypass(bypassed) 219 + } 220 + return bypassed; 221 + } 222 + export const rejoinBypassed = (str: string): string => { 223 + let bypassed: string = str; 224 + for(const b of bypassJoinerMap) { 225 + bypassed = b.rejoin(bypassed) 226 + } 227 + return bypassed; 184 228 } 185 229 export const containsDelimiters = (str: string) => null !== str.match(/[,&/\\]+/i) 186 230 export const findDelimiters = (str: string) => {

+2 -2

src/core/Atomic.ts

··· 254 254 play: PlayObject, 255 255 playFirstSeenAt?: string, 256 256 playLastUpdatedAt?: string, 257 - playerLastUpdatedAt: strin 257 + playerLastUpdatedAt: string 258 258 position?: Second 259 259 listenedDuration: Second 260 260 status: { ··· 345 345 } 346 346 347 347 export type Joiner = ',' | '&' | '/' | '\\' | string; 348 - export const JOINERS: Joiner[] = [',','&','/','\\']; 348 + export const JOINERS: Joiner[] = [',','/','\\']; 349 349 350 350 export type FinalJoiners = '&'; 351 351 export const JOINERS_FINAL: FinalJoiners[] = ['&'];

Configure Feed

Configure Feed