[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
feat(database): Implement restore from backup
+49
-33
+1
.gitignore
+1
.gitignore
+6
src/backend/common/database/Database.ts
+6
src/backend/common/database/Database.ts
···
23
23
return path.resolve(workingDirectory ?? configDir, `${name}`);
24
24
}
25
25
26
+
export const getDbBackupPath = (dbPath: string, suffix?: string): string => {
27
+
const pathInfo = path.parse(dbPath);
28
+
const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}.bak${suffix !== undefined ? `.${suffix}` : ''}`;
29
+
return backupPath;
30
+
}
31
+
26
32
export const backupDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise<void> => {
27
33
28
34
const {
+24
-14
src/backend/common/database/drizzle/drizzleUtils.ts
+24
-14
src/backend/common/database/drizzle/drizzleUtils.ts
···
3
3
import { migrate } from 'drizzle-orm/node-sqlite/migrator';
4
4
import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator';
5
5
import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
6
-
import { PGlite } from '@electric-sql/pglite';
6
+
import { PGlite, PGliteOptions } from '@electric-sql/pglite';
7
7
import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm';
8
8
import * as fs from 'fs/promises';
9
+
import * as fsSync from 'fs';
9
10
import * as path from 'path';
10
-
import { backupDb, getDbPath, MEMORY_DB_NAME } from '../Database.js';
11
+
import { backupDb, getDbBackupPath, getDbPath, MEMORY_DB_NAME } from '../Database.js';
11
12
import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js';
12
13
import { childLogger, Logger, LogLevel } from '@foxxmd/logging';
13
14
import { loggerNoop } from '../../MaybeLogger.js';
···
30
31
logger.info(`No database exists, no backup needed.`);
31
32
return [false, []];
32
33
}
33
-
db = getDb(dbVal);
34
+
db = await getDb(dbVal);
34
35
} else {
35
36
db = dbVal;
36
37
}
···
87
88
}
88
89
}
89
90
90
-
export const getDb = (dbVal: string | PGlite, opts: { logger?: Logger } = {}) => {
91
+
export const getDb = async (dbVal: string | PGlite, opts: { logger?: Logger, backupPath?: string } = {}) => {
91
92
const {
92
93
logger = loggerNoop,
94
+
backupPath,
93
95
} = opts;
94
96
let client: PGlite;
95
97
96
98
if(typeof dbVal === 'string') {
97
-
const dbPath = dbVal;
98
-
99
-
if(dbVal === MEMORY_DB_NAME) {
100
-
client = new PGlite();
101
-
} else {
102
-
client = new PGlite(dbPath);
99
+
const opts: PGliteOptions = {};
100
+
if(dbVal !== MEMORY_DB_NAME) {
101
+
opts.dataDir = dbVal;
102
+
if(backupPath !== undefined) {
103
+
opts.loadDataDir = new Blob([fsSync.readFileSync(backupPath)]);
104
+
}
103
105
}
106
+
client = await PGlite.create(opts);
104
107
} else {
105
108
client = dbVal;
106
109
}
107
110
return drizzlePglite({relations: relations, logger: createDrizzleLogger(logger), client});
108
111
}
109
112
110
-
export type DbConcrete = ReturnType<typeof getDb>;
113
+
export type DbConcrete = Awaited<ReturnType<typeof getDb>>;
111
114
112
115
export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
113
116
const {
···
155
158
throw new Error('Database directory is not accessible', { cause: e });
156
159
}
157
160
158
-
db = getDb(dbPath, opts);
161
+
const backupPath = getDbBackupPath(dbPath);
159
162
160
163
if (fileExists(dbPath)) {
161
-
164
+
db = await getDb(dbPath, opts);
162
165
const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts);
163
166
if (shouldBackup) {
164
167
hasPendingMigrations = true;
165
168
await backupPgDb(db, dbPath, { logger: opts.logger });
166
169
}
170
+
} else if(fileExists(backupPath)) {
171
+
logger.info(`Detected no database, using backup to recreate db. Backup file: ${backupPath}`);
172
+
db = await getDb(dbPath, {...opts, backupPath});
173
+
const usedBackedPath = getDbBackupPath(dbPath, 'used');
174
+
logger.info(`Backup loaded! Renaming backup to indicate it has already been used, new path: ${usedBackedPath}`);
175
+
await fs.rename(backupPath, usedBackedPath);
167
176
} else {
168
177
logger.info('Detected no database, creating a new one...');
178
+
db = await getDb(dbPath);
169
179
isNew = true;
170
180
}
171
181
} else {
172
182
logger.info('Detected in-memory database');
173
-
db = getDb(dbPath, opts);
183
+
db = await getDb(dbPath, opts);
174
184
isNew = true;
175
185
}
176
186
+3
-3
src/backend/common/database/drizzle/repositories/BaseRepository.ts
+3
-3
src/backend/common/database/drizzle/repositories/BaseRepository.ts
···
1
1
import { childLogger, Logger } from "@foxxmd/logging";
2
-
import { getDb } from "../drizzleUtils.js";
2
+
import { DbConcrete } from "../drizzleUtils.js";
3
3
import { CompareOpKey } from "../drizzleTypes.js";
4
4
import { Dayjs } from "dayjs";
5
5
import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
···
41
41
displayName: string;
42
42
tableName: TableName;
43
43
table: ReturnType<typeof getConfigByTableName<T>>
44
-
db: ReturnType<typeof getDb>;
44
+
db: DbConcrete;
45
45
componentId?: number
46
46
47
-
constructor(db: ReturnType<typeof getDb>, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
47
+
constructor(db: DbConcrete, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
48
48
this.db = db;
49
49
this.displayName = displayName;
50
50
this.tableName = tableName;
+2
-2
src/backend/common/database/drizzle/repositories/ComponentRepository.ts
+2
-2
src/backend/common/database/drizzle/repositories/ComponentRepository.ts
···
1
1
import { Logger } from "drizzle-orm";
2
2
import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
3
-
import { getDb } from "../drizzleUtils.js";
3
+
import { DbConcrete } from "../drizzleUtils.js";
4
4
import { ComponentNew, ComponentSelect, FindWhere } from "../drizzleTypes.js";
5
5
import { components } from "../schema/schema.js";
6
6
import { generateComponentEntity } from "../entityUtils.js";
7
7
8
8
export class DrizzleComponentRepository extends DrizzleBaseRepository<'components'> {
9
9
10
-
constructor(db: ReturnType<typeof getDb>, opts: DrizzleRepositoryOpts = {}) {
10
+
constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
11
11
super(db, 'components', 'Component', opts);
12
12
}
13
13
+2
-2
src/backend/common/database/drizzle/repositories/PlayRepository.ts
+2
-2
src/backend/common/database/drizzle/repositories/PlayRepository.ts
···
1
1
import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging";
2
-
import { DbConcrete, getDb, runTransaction } from "../drizzleUtils.js";
2
+
import { DbConcrete } from "../drizzleUtils.js";
3
3
import { loggerNoop } from "../../../MaybeLogger.js";
4
4
import { ErrorLike, PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../../../../core/Atomic.js";
5
5
import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js";
···
69
69
protected getQueueNextPrepared?: ReturnType<typeof this.prepareGetQueueNext>
70
70
protected getQueuedScrobbleRangePrepared?: ReturnType<typeof this.prepareGetQueuedScrobbleRange>
71
71
72
-
constructor(db: ReturnType<typeof getDb>, opts: DrizzleRepositoryOpts = {}) {
72
+
constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
73
73
super(db, 'plays', 'Plays', opts);
74
74
}
75
75
+2
-2
src/backend/common/database/drizzle/repositories/QueueRepository.ts
+2
-2
src/backend/common/database/drizzle/repositories/QueueRepository.ts
···
1
1
import { eq, and, lte, inArray } from "drizzle-orm";
2
2
import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
3
-
import { getDb } from "../drizzleUtils.js";
3
+
import { DbConcrete } from "../drizzleUtils.js";
4
4
import { QueueStateSelect } from "../drizzleTypes.js";
5
5
import { queueStates } from "../schema/schema.js";
6
6
import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.js";
7
7
export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> {
8
8
9
-
constructor(db: ReturnType<typeof getDb>, opts: DrizzleRepositoryOpts = {}) {
9
+
constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
10
10
super(db, 'queueStates', 'Queue', opts);
11
11
}
12
12
+8
-8
src/backend/tests/database/drizzle.test.ts
+8
-8
src/backend/tests/database/drizzle.test.ts
···
37
37
38
38
it('Detects abnormal db', async function () {
39
39
// database exists but there is no __drizzle_migrations table
40
-
const db = getDb((await getPrepopulatedMemoryPGlite()));
40
+
const db = await getDb((await getPrepopulatedMemoryPGlite()));
41
41
const [shouldBackup, pending] = await shouldBackupDb(db);
42
42
expect(shouldBackup).is.true;
43
43
expect(pending).length(0);
···
57
57
try {
58
58
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
59
59
const mf = path.resolve('./migrations');
60
-
const db = getDb((await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))));
60
+
const db = await getDb((await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))));
61
61
await migrateDb(db, { migrationsFolder: mf });
62
62
const res = await x('drizzle-kit', [
63
63
'generate',
···
95
95
try {
96
96
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
97
97
const mf = path.resolve('./migrations');
98
-
const db = getDb((await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))));
98
+
const db = await getDb((await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd()))));
99
99
await migrateDb(db, { migrationsFolder: mf });
100
100
const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf });
101
101
expect(shouldBackup).is.false;
···
121
121
try {
122
122
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
123
123
const mf = path.resolve('./migrations');
124
-
const db = getDb((await getPrepopulatedFSPGlite(dbPath)));
124
+
const db = await getDb((await getPrepopulatedFSPGlite(dbPath)));
125
125
await migrateDb(db, { migrationsFolder: mf });
126
126
const res = await x('drizzle-kit', [
127
127
'generate',
···
560
560
561
561
await withLocalTmpDir(async () => {
562
562
try {
563
-
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
563
+
let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
564
564
await migrateDb(db);
565
565
const play100Component = Number.parseInt((await x('du', ['-ksb','.'])).stdout.split('\t')[0]);
566
566
loggerDebug.debug(`100 Plays => ${formatNumber((play100Component / 1024) / 1024, {toFixed: 2})}mb`);
···
576
576
577
577
await withLocalTmpDir(async () => {
578
578
try {
579
-
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
579
+
let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
580
580
await migrateDb(db);
581
581
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
582
582
···
611
611
612
612
await withLocalTmpDir(async () => {
613
613
try {
614
-
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
614
+
let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
615
615
await migrateDb(db);
616
616
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
617
617
···
645
645
646
646
await withLocalTmpDir(async () => {
647
647
try {
648
-
let db = getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
648
+
let db = await getDb(await getPrepopulatedFSPGlite(getDbPath('msDb', process.cwd())));
649
649
await migrateDb(db);
650
650
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
651
651
-1
src/backend/tests/setup.ts
-1
src/backend/tests/setup.ts
···
1
1
import { loggerTest } from '@foxxmd/logging';
2
2
import { getRoot } from "../ioc.js";
3
3
import { transientCache, transientDb } from './utils/TransientTestUtils.js';
4
-
import { DbConcrete, getDb, migrateDb } from '../common/database/drizzle/drizzleUtils.js';
5
4
6
5
// let transientD: DbConcrete;
7
6
// const transientDbFactory = () => {
+1
-1
src/backend/tests/utils/TransientTestUtils.ts
+1
-1
src/backend/tests/utils/TransientTestUtils.ts
···
11
11
export const transientDb = async () => {
12
12
if(baseDb === undefined) {
13
13
baseDb = await getPrepopulatedMemoryPGlite();
14
-
await migrateDb(getDb(baseDb));
14
+
await migrateDb(await getDb(baseDb));
15
15
}
16
16
const db = getDb((await baseDb.clone()) as Awaited<PGlite>);
17
17
return db;