[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
feat: Improve cache testing and cleanup cache serialization
* Remove ListenProgress specific serialization tests and function since its not needed anymore
* postfix tmp dirs with test name hints for easier debugging
* try to prevent filecache from leaving files behind
+42
-107
+1
-4
src/backend/common/Cache.ts
+1
-4
src/backend/common/Cache.ts
···
19
19
import { Typeson } from 'typeson';
20
20
import { builtin } from 'typeson-registry';
21
21
import { loggerNoop } from './MaybeLogger.js';
22
-
import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js';
23
22
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
24
23
import prom, { Gauge } from 'prom-client';
25
24
import { nonEmptyStringOrDefault } from '../../core/StringUtils.js';
···
39
38
(x) => dayjs.isDayjs(x),
40
39
(d: Dayjs) => d.toJSON(),
41
40
(date) => dayjs(date)
42
-
],
43
-
ListenProgressTS,
44
-
ListenProgressPositional
41
+
]
45
42
});
46
43
47
44
const unsupportedEnvKeys = [
+16
-92
src/backend/tests/cache/cache.test.ts
+16
-92
src/backend/tests/cache/cache.test.ts
···
4
4
import { after, before, describe, it } from 'mocha';
5
5
import dayjs from "dayjs";
6
6
import withLocalTmpDir from 'with-local-tmp-dir';
7
-
import { initFileCache, initMemoryCache, initValkeyCache, MSCache } from "../../common/Cache.js";
8
-
import { generatePlays } from "../../../core/PlayTestUtils.js";
7
+
import { initFileCache, initMemoryCache, initValkeyCache } from "../../common/Cache.js";
9
8
import { ListenProgressPositional, ListenProgressTS } from "../../sources/PlayerState/ListenProgress.js";
10
9
import { isPortReachableConnect } from "../../utils/NetworkUtils.js";
11
-
import { getRoot } from "../../ioc.js";
12
-
import { transientCache } from "../utils/TransientTestUtils.js";
13
-
import { TestScrobbler } from "../scrobbler/TestScrobbler.js";
14
10
import { sleep } from "../../utils.js";
15
-
import {promises} from 'node:fs';
16
11
17
12
chai.use(asPromised);
18
13
···
56
51
57
52
it('File cache serializes and deserializes dayjs', async function () {
58
53
59
-
withLocalTmpDir(async () => {
54
+
await withLocalTmpDir(async () => {
55
+
// for some reason this *recreates* an empty cache after the test has finished (when running the full suite)
56
+
// i think its because of long persist interval compared to test time?
57
+
//
58
+
// so:
59
+
// make intervals very small
60
+
// call destroy on both cache instances (shouldn't be necessary)
61
+
// sleep for longer than persist interal so tmp dir callback can (hopefully) properly delete any files
60
62
61
-
const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() });
63
+
const [keyv, flat] = await initFileCache({ cacheDir: process.cwd(), persistInterval: 5, expirationInterval: 4 });
62
64
63
65
const now = dayjs();
64
66
65
67
await keyv.set('foo', now);
66
-
flat.save();
68
+
flat.save(true);
69
+
keyv.disconnect();
67
70
68
-
const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() });
71
+
const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd(), persistInterval: 5, expirationInterval: 4 });
69
72
70
73
const time = await cleanKeyv.get('foo');
71
74
···
73
76
expect(time instanceof dayjs).is.true;
74
77
expect(now.toJSON()).eq((time as any).toJSON());
75
78
flat.destroy();
76
-
77
-
}, { unsafeCleanup: true });
78
-
});
79
-
80
-
it('File cache serializes and deserializes ListenProgress', async function () {
81
-
82
-
withLocalTmpDir(async () => {
83
-
84
-
const [keyv, flat] = await initFileCache({ cacheDir: process.cwd() });
85
-
86
-
const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 });
87
-
88
-
await keyv.set('foo', prog);
89
-
await flat.save();
90
-
91
-
const [cleanKeyv, cleanFlat] = await initFileCache({ cacheDir: process.cwd() });
92
-
93
-
const cachedProg = await cleanKeyv.get('foo');
94
-
95
-
expect(cachedProg).to.not.be.undefined;
96
-
expect(cachedProg instanceof ListenProgressTS).is.true;
97
-
expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON());
98
-
flat.destroy();
99
-
100
-
}, { unsafeCleanup: true });
79
+
cleanFlat.destroy();
80
+
await sleep(10);
81
+
}, { unsafeCleanup: true, postfix: 'fileCacheDajys' });
101
82
});
102
83
});
103
84
···
127
108
expect(now.toJSON()).eq((time as any).toJSON());
128
109
129
110
});
130
-
131
-
it('Valkey cache serializes and deserializes ListenProgress', async function () {
132
-
133
-
const keyv = await initValkeyCache('test', 'redis://valkey:6379');
134
-
await keyv.clear();
135
-
136
-
const prog = new ListenProgressPositional({ timestamp: dayjs(), position: 35, positionPercent: 50 });
137
-
138
-
await keyv.set('foo', prog);
139
-
140
-
const cachedProg = await keyv.get('foo');
141
-
142
-
expect(cachedProg).to.not.be.undefined;
143
-
expect(cachedProg instanceof ListenProgressTS).is.true;
144
-
expect(cachedProg.timestamp.toJSON()).eq(prog.timestamp.toJSON());
145
-
146
-
});
147
111
});
148
-
149
-
// describe('#ScrobbleCache', function () {
150
-
151
-
// afterEach(function () {
152
-
// const root = getRoot();
153
-
// root.upsert({ cache: () => transientCache });
154
-
// root.items.cache().init();
155
-
// });
156
-
157
-
// it('Preserves scrobbles', async function () {
158
-
159
-
// this.timeout(10000);
160
-
161
-
// // why does this take so long?
162
-
// await withLocalTmpDir(async () => {
163
-
164
-
// const root = getRoot();
165
-
// root.upsert({ cache: () => () => new MSCache(loggerTest, { scrobble: { provider: 'file', connection: process.cwd(), persistInterval: 100 } }) });
166
-
167
-
// await using test = new TestScrobbler();
168
-
// await test.initialize();
169
-
// const plays = generatePlays(100, {}, {}, {listenRanges: true});
170
-
// await test.queueScrobble(plays, 'testSource');
171
-
// const queued = test.queuedScrobbles.map(x => x.play);
172
-
// await sleep(101);
173
-
// const dirContents = await promises.readdir('.');
174
-
// const hasCache = dirContents.some(x => x === 'ms-scrobble.cache');
175
-
// expect(hasCache).is.true;
176
-
177
-
// await using newTest = new TestScrobbler();
178
-
// await newTest.initialize();
179
-
// expect(newTest.queuedScrobbles.length).to.eq(plays.length);
180
-
// expect(newTest.queuedScrobbles[0].play.data.track).to.eq(queued[0].data.track);
181
-
182
-
// }, { unsafeCleanup: true });
183
-
184
-
// });
185
-
186
-
// });
187
-
188
112
});
+2
-2
src/backend/tests/config/config.test.ts
+2
-2
src/backend/tests/config/config.test.ts
···
44
44
let reset: any;
45
45
46
46
beforeEach(async function() {
47
-
reset = await withLocalTmpDir({unsafeCleanup: true});
47
+
reset = await withLocalTmpDir({unsafeCleanup: true, postfix: 'sourceConfigParse'});
48
48
});
49
49
50
50
afterEach(async function() {
···
78
78
let reset: any;
79
79
80
80
beforeEach(async function() {
81
-
reset = await withLocalTmpDir({unsafeCleanup: true});
81
+
reset = await withLocalTmpDir({unsafeCleanup: true, postfix: 'clientConfigParse'});
82
82
});
83
83
84
84
afterEach(async function() {
+9
-9
src/backend/tests/database/drizzle.test.ts
+9
-9
src/backend/tests/database/drizzle.test.ts
···
31
31
const [shouldBackup, pending] = await shouldBackupDb(getDbPath('notreal', process.cwd()));
32
32
expect(shouldBackup).is.false;
33
33
expect(pending).length(0);
34
-
});
34
+
}, {postfix: 'noDb'});
35
35
36
36
});
37
37
···
43
43
expect(shouldBackup).is.true;
44
44
expect(pending).length(0);
45
45
otherdb.close();
46
-
}, { unsafeCleanup: true });
46
+
}, { unsafeCleanup: true, postfix: 'badDb' });
47
47
48
48
});
49
49
···
82
82
} catch (e) {
83
83
throw e;
84
84
}
85
-
}, { unsafeCleanup: true });
85
+
}, { unsafeCleanup: true, postfix: 'pendingMigrations' });
86
86
});
87
87
88
88
it('Detects no pending migrations correctly', async function () {
···
107
107
} catch (e) {
108
108
throw e;
109
109
}
110
-
}, { unsafeCleanup: true });
110
+
}, { unsafeCleanup: true, postfix: 'noMigrations' });
111
111
});
112
112
113
113
it('Backs up database when migrations are pending', async function () {
···
149
149
} catch (e) {
150
150
throw e;
151
151
}
152
-
}, { unsafeCleanup: true });
152
+
}, { unsafeCleanup: true, postfix: 'dbBackup' });
153
153
});
154
154
155
155
});
···
572
572
} catch (e) {
573
573
throw e;
574
574
}
575
-
}, { unsafeCleanup: true });
575
+
}, { unsafeCleanup: true, postfix: 'dbStatEmpty' });
576
576
});
577
577
578
578
it('get db plays size stats', async function () {
···
597
597
} catch (e) {
598
598
throw e;
599
599
}
600
-
}, { unsafeCleanup: true });
600
+
}, { unsafeCleanup: true, postfix: 'dbStatPlain' });
601
601
});
602
602
603
603
it('get db plays size stats with input', async function () {
···
622
622
} catch (e) {
623
623
throw e;
624
624
}
625
-
}, { unsafeCleanup: true });
625
+
}, { unsafeCleanup: true, postfix: 'dbStatInput' });
626
626
});
627
627
628
628
it('get db plays size stats with input and lifecycle', async function () {
···
647
647
} catch (e) {
648
648
throw e;
649
649
}
650
-
}, { unsafeCleanup: true });
650
+
}, { unsafeCleanup: true, postfix: 'dbStatAll' });
651
651
});
652
652
})
+3
src/core/Atomic.ts
+3
src/core/Atomic.ts
···
605
605
* It needs to be cheap since we mostly use this when walking play objects to transform strings back to dayjs and there may be many strings to check
606
606
*/
607
607
export const REGEX_ISO8601_LOOSE = new RegExp(/\d{4}-[01]\d-[0-3]\dT/);
608
+
/** A string we previously marshalled has a wellknown prefix and only check for DateT since we can reasonbly sure if this exists its a date we can parse with dayjs
609
+
*/
610
+
export const REGEX_ISO8601_WELLKNOWN = new RegExp(/dayjs-(\d{4}-[01]\d-[0-3]\dT.*)/);
608
611
609
612
export const CLIENT_INGRESS_QUEUE = 'ingress';
610
613
export const CLIENT_DEAD_QUEUE = 'dead';
+11
src/core/PlayMarshalUtils.ts
+11
src/core/PlayMarshalUtils.ts
···
101
101
return data as unknown as PlayObject;
102
102
};
103
103
104
+
export const marshalDayjsToWellKnownString = (ctx: TraverseContext, x: any): void => {
105
+
if (dayjs.isDayjs(x)) {
106
+
ctx.update(`dayjs-x.toISOString()`);
107
+
}
108
+
}
109
+
110
+
export const marshalWellKnownIsoStringToDayjs = (ctx: TraverseContext, x: any): void => {
111
+
if (typeof x === 'string' && REGEX_ISO8601_LOOSE.test(x)) {
112
+
ctx.update(dayjs(x.substring(6)), true);
113
+
}
114
+
}