[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
all tests passing and migrate/backup works
+92
-83
+63
-53
src/backend/common/database/drizzle/drizzleUtils.ts
+63
-53
src/backend/common/database/drizzle/drizzleUtils.ts
···
15
15
import { relations } from './schema/schema.js';
16
16
import { addToContext, executeQuery } from './logContext.js';
17
17
18
-
export async function shouldBackupDb(db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> {
18
+
export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> {
19
19
const {
20
20
logger: parentLogger = loggerNoop,
21
21
migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')
22
22
} = opts;
23
23
const logger = childLogger(parentLogger, 'Migrations');
24
+
25
+
let db: DbConcrete;
24
26
25
-
// logger.info(`Checking database at ${dbPath}`);
26
-
// if (dbPath !== MEMORY_DB_NAME && !fileExists(dbPath)) {
27
-
// logger.info(`No database exists!`);
28
-
// return [false, []];
29
-
// }
27
+
if(typeof dbVal === 'string') {
28
+
logger.info(`Checking for database at ${dbVal}`);
29
+
if (dbVal !== MEMORY_DB_NAME && !fileExists(dbVal)) {
30
+
logger.info(`No database exists, no backup needed.`);
31
+
return [false, []];
32
+
}
33
+
db = getDb(dbVal);
34
+
} else {
35
+
db = dbVal;
36
+
}
37
+
30
38
31
39
// const db = drizzlePglite(dbPath);
32
40
···
79
87
}
80
88
}
81
89
82
-
export const getDb = (dbName: string | PGlite = 'msDb', opts: { logger?: Logger, workingDirectory?: string } = {}) => {
90
+
export const getDb = (dbVal: string | PGlite, opts: { logger?: Logger } = {}) => {
83
91
const {
84
-
workingDirectory,
85
92
logger = loggerNoop,
86
93
} = opts;
87
94
let client: PGlite;
88
95
89
-
if(typeof dbName === 'string') {
90
-
const dbPath = getDbPath(dbName, workingDirectory);
96
+
if(typeof dbVal === 'string') {
97
+
const dbPath = dbVal;
91
98
92
-
if(dbName === MEMORY_DB_NAME) {
99
+
if(dbVal === MEMORY_DB_NAME) {
93
100
client = new PGlite();
94
101
} else {
95
102
client = new PGlite(dbPath);
96
103
}
97
104
} else {
98
-
client = dbName;
105
+
client = dbVal;
99
106
}
100
107
return drizzlePglite({relations: relations, logger: createDrizzleLogger(logger), client});
101
108
}
102
109
103
110
export type DbConcrete = ReturnType<typeof getDb>;
104
111
105
-
export const migrateDb = async (db: ReturnType<typeof drizzlePglite>, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
112
+
export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
106
113
const {
107
114
migrationsFolder,
108
115
logger: parentLogger = loggerNoop
···
134
141
}
135
142
}
136
143
137
-
export const performDbMigrationWithBackup = async (dbName: string = 'msDb', opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string } = {}) => {
138
-
const dbPath = getDbPath(dbName, opts.workingDirectory);
144
+
export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string } = {}): Promise<[DbConcrete, boolean]> => {
145
+
const {
146
+
logger = loggerNoop
147
+
} = opts;
148
+
let db: DbConcrete,
149
+
isNew = false,
150
+
hasPendingMigrations: boolean = true;
151
+
if (dbPath !== MEMORY_DB_NAME) {
152
+
try {
153
+
fileOrDirectoryIsWriteable(dbPath);
154
+
} catch (e) {
155
+
throw new Error('Database directory is not accessible', { cause: e });
156
+
}
139
157
140
-
if(fileExists(dbPath)) {
141
-
const [shouldBackup, pendingMigrations] = await shouldBackupDb(dbPath, opts);
142
-
if(shouldBackup) {
143
-
await backupPgDb(dbName, opts);
158
+
db = getDb(dbPath, opts);
159
+
160
+
if (fileExists(dbPath)) {
161
+
162
+
const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts);
163
+
if (shouldBackup) {
164
+
hasPendingMigrations = true;
165
+
await backupPgDb(db, dbPath, { logger: opts.logger });
166
+
}
167
+
} else {
168
+
logger.info('Detected no database, creating a new one...');
169
+
isNew = true;
144
170
}
171
+
} else {
172
+
logger.info('Detected in-memory database');
173
+
db = getDb(dbPath, opts);
174
+
isNew = true;
145
175
}
146
176
177
+
if(hasPendingMigrations && dbPath !== MEMORY_DB_NAME) {
178
+
logger.info('TIP: Migrations may take some time, depending on the size of your database');
179
+
}
180
+
await migrateDb(db, opts);
147
181
148
-
const db = getDb(dbName, opts);
149
-
await migrateDb(db, opts);
182
+
return [db, isNew];
150
183
}
151
184
152
-
export const backupPgDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise<void> => {
185
+
export const backupPgDb = async (db: DbConcrete, dbPath: string, opts: { logger?: Logger } = {}): Promise<void> => {
153
186
154
187
const {
155
188
logger: parentLogger = loggerNoop,
156
-
workingDirectory
157
189
} = opts;
158
190
159
-
const logger = childLogger(parentLogger, 'Migrations');
160
-
161
-
const dbPath = getDbPath(dbName, workingDirectory);
162
-
let newDb = false;
163
-
164
-
if (dbPath !== MEMORY_DB_NAME) {
165
-
if (!fileExists(dbPath)) {
166
-
logger.info(`Database at ${dbPath} does not exist, will create it.`);
167
-
newDb = true;
168
-
}
169
-
try {
170
-
fileOrDirectoryIsWriteable(dbPath);
171
-
} catch (e) {
172
-
throw new Error('Database path/folder is not writeable, cannot backup database', { cause: e });
173
-
}
174
-
}
191
+
const logger = childLogger(parentLogger, 'Backup');
175
192
176
-
if (dbPath !== MEMORY_DB_NAME && !newDb) {
177
-
178
-
let client: PGlite;
179
-
if(dbName === MEMORY_DB_NAME) {
180
-
client = new PGlite();
181
-
} else {
182
-
client = new PGlite(dbName);
183
-
}
184
-
const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`;
185
-
logger.info(`Backing up database before migrating => ${backupPath}`);
186
-
fs.writeFile(backupPath, Buffer.from(await (await client.dumpDataDir()).arrayBuffer()));
187
-
await fs.copyFile(dbPath, backupPath)
188
-
logger.info('Backed up!');
189
-
}
193
+
const pathInfo = path.parse(dbPath);
194
+
// being extra sure there isn't a trailing slash
195
+
const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}-${Date.now()}.bak`;
196
+
logger.info(`Backing up database before migrating => ${backupPath}`);
197
+
fs.writeFile(backupPath, Buffer.from(await (await db.$client.dumpDataDir()).arrayBuffer()));
198
+
//await fs.copyFile(dbPath, backupPath)
199
+
logger.info('Backed up!');
190
200
}
191
201
192
202
export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => {
+5
-4
src/backend/index.ts
+5
-4
src/backend/index.ts
···
21
21
import ScrobbleClients from './scrobblers/ScrobbleClients.js';
22
22
import ScrobbleSources from './sources/ScrobbleSources.js';
23
23
import { Notifiers } from './notifier/Notifiers.js';
24
-
import { getDb, performDbMigrationWithBackup } from './common/database/drizzle/drizzleUtils.js';
24
+
import { getMigratedDb } from './common/database/drizzle/drizzleUtils.js';
25
25
import { getDbPath } from './common/database/Database.js';
26
26
import { createRetentionCleanupTask } from './tasks/retentionCleanup.js';
27
27
import { parseUserConfig } from './common/Cache.js';
···
98
98
const [aLogger, appLoggerStream] = await appLogger(logging)
99
99
logger = childLogger(aLogger, 'App');
100
100
101
-
logger.info(`Using database at ${getDbPath('ms')}`);
102
-
await performDbMigrationWithBackup('ms', {logger});
101
+
const dbPath = getDbPath('msDb');
102
+
logger.info(`Using database at ${getDbPath('msDb')}`);
103
+
const [db, isNew] = await getMigratedDb(dbPath, {logger: childLogger(logger, 'DB')});
103
104
104
105
const root = getRoot({
105
106
...config,
···
107
108
logger,
108
109
loggingConfig: logging,
109
110
loggerStream: appLoggerStream,
110
-
db: getDb('ms', {logger})
111
+
db
111
112
});
112
113
113
114
const internalConfigOptional = {
+24
-26
src/backend/tests/database/drizzle.test.ts
+24
-26
src/backend/tests/database/drizzle.test.ts
···
1
1
import chai, { assert, expect } from 'chai';
2
2
import asPromised from 'chai-as-promised';
3
-
import { getDb, migrateDb, performDbMigrationWithBackup, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js';
3
+
import { getDb, getMigratedDb, migrateDb, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js';
4
4
import withLocalTmpDir from 'with-local-tmp-dir';
5
5
import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js';
6
6
import dayjs from 'dayjs';
···
26
26
27
27
describe('Migrations', function () {
28
28
29
-
// it('Detects non-existent db', async function () {
29
+
it('Detects non-existent db', async function () {
30
30
31
-
// await withLocalTmpDir(async () => {
32
-
// const [shouldBackup, pending] = await shouldBackupDb(getDbPath('notreal', process.cwd()));
33
-
// expect(shouldBackup).is.false;
34
-
// expect(pending).length(0);
35
-
// }, {postfix: 'noDb'});
31
+
await withLocalTmpDir(async () => {
32
+
const [db, isNew] = await getMigratedDb(getDbPath('notreal', process.cwd()));
33
+
expect(isNew).is.true;
34
+
}, {postfix: 'noDb'});
36
35
37
-
// });
38
-
39
-
// it('Detects abnormal db', async function () {
36
+
});
40
37
41
-
// await withLocalTmpDir(async () => {
42
-
// const otherdb = new DatabaseSync(path.resolve('./', 'other.db'));
43
-
// const [shouldBackup, pending] = await shouldBackupDb(getDbPath('other', process.cwd()));
44
-
// expect(shouldBackup).is.true;
45
-
// expect(pending).length(0);
46
-
// otherdb.close();
47
-
// }, { unsafeCleanup: true, postfix: 'badDb' });
38
+
it('Detects abnormal db', async function () {
39
+
// database exists but there is no __drizzle_migrations table
40
+
const db = getDb((await getPrepopulatedMemoryPGlite()));
41
+
const [shouldBackup, pending] = await shouldBackupDb(db);
42
+
expect(shouldBackup).is.true;
43
+
expect(pending).length(0);
48
44
49
-
// });
45
+
});
50
46
51
47
it('Detects pending migrations', async function () {
52
48
···
119
115
120
116
await withLocalTmpDir(async () => {
121
117
118
+
const dbPath = getDbPath('msDb', process.cwd());
122
119
// copy first migration
123
120
await fs.mkdir('migrations');
124
121
try {
125
122
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
126
123
const mf = path.resolve('./migrations');
127
-
const db = getDb('ms', { workingDirectory: process.cwd() });
124
+
const db = getDb((await getPrepopulatedFSPGlite(dbPath)));
128
125
await migrateDb(db, { migrationsFolder: mf });
129
126
const res = await x('drizzle-kit', [
130
127
'generate',
···
136
133
'--schema',
137
134
path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
138
135
'--dialect',
139
-
'sqlite'
136
+
'postgresql'
140
137
]);
141
138
db.$client.close();
142
139
···
144
141
const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration'));
145
142
await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`);
146
143
147
-
await performDbMigrationWithBackup('ms', {workingDirectory: process.cwd(), migrationsFolder: mf});
144
+
await getMigratedDb(dbPath, {workingDirectory: process.cwd(), migrationsFolder: mf})
148
145
const contents = await fs.readdir(path.resolve('./'));
149
-
expect(contents.some(x => x.includes('ms.db.bak')));
146
+
const backupPattern = new RegExp(/msDb-\d+\.bak/)
147
+
expect(contents.some(x => backupPattern.test(x))).is.true;
150
148
} catch (e) {
151
149
throw e;
152
150
}
···
562
560
563
561
await withLocalTmpDir(async () => {
564
562
try {
565
-
let db = getDb('msDb', { workingDirectory: process.cwd() });
563
+
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
566
564
await migrateDb(db);
567
565
const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
568
566
loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`);
···
578
576
579
577
await withLocalTmpDir(async () => {
580
578
try {
581
-
let db = getDb((await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))));
579
+
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
582
580
await migrateDb(db);
583
581
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
584
582
···
613
611
614
612
await withLocalTmpDir(async () => {
615
613
try {
616
-
let db = getDb('msDb', { workingDirectory: process.cwd() });
614
+
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
617
615
await migrateDb(db);
618
616
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
619
617
···
647
645
648
646
await withLocalTmpDir(async () => {
649
647
try {
650
-
let db = getDb('msDb', { workingDirectory: process.cwd() });
648
+
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
651
649
await migrateDb(db);
652
650
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
653
651