[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients
docs.multi-scrobbler.app
deezer
docker
jellyfin
koito
lastfm
listenbrainz
maloja
mopidy
mpris
music
music-assistant
plex
scrobble
self-hosted
spotify
subsonic
tautulli
youtube-music
port changes from pgllite branch
Lots of cleaned up methods and better framing if db backend ever becomes async again
+226
-137
+1
-1
drizzle.config.ts
+1
-1
drizzle.config.ts
+8
-8
package-lock.json
+8
-8
package-lock.json
···
47
47
"dbus-ts": "^0.0.7",
48
48
"discord.js": "^14.26.0",
49
49
"dotenv": "^10.0.0",
50
-
"drizzle-orm": "^1.0.0-rc.2-8feace6",
50
+
"drizzle-orm": "^1.0.0-rc.2-c5a84d1",
51
51
"express": "^5.2.1",
52
52
"express-session": "^1.19.0",
53
53
"fast-equals": "^6.0.0",
···
146
146
"chai": "^4.3.6",
147
147
"chai-as-promised": "^8.0.2",
148
148
"clsx": "^2.1.1",
149
-
"drizzle-kit": "^1.0.0-rc.1-929a083",
149
+
"drizzle-kit": "^1.0.0-rc.2-c5a84d1",
150
150
"eslint": "^10.3.0",
151
151
"eslint-plugin-prefer-arrow-functions": "^3.2.4",
152
152
"eslint-plugin-storybook": "^10.3.6",
···
8251
8251
}
8252
8252
},
8253
8253
"node_modules/drizzle-kit": {
8254
-
"version": "1.0.0-rc.1-929a083",
8255
-
"resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-rc.1-929a083.tgz",
8256
-
"integrity": "sha512-Bo7T8db9V+nYC5wQjNsHgGKyxZWJkMUVUKVqveEtKxtmV3opCoG1FoHEcRTrAAz+CBzCkyM82VG2aObT229oxw==",
8254
+
"version": "1.0.0-rc.2-e38a2ba",
8255
+
"resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-1.0.0-rc.2-e38a2ba.tgz",
8256
+
"integrity": "sha512-2A+OcNPBBVHe9QKZRkOE8C0PsH9x10QlZqYicEJTh+XDnNBL9tdMTTJ4eSjHG8CBegXnUy4h3WmsRcgO1S8eOA==",
8257
8257
"dev": true,
8258
8258
"license": "MIT",
8259
8259
"dependencies": {
···
8752
8752
}
8753
8753
},
8754
8754
"node_modules/drizzle-orm": {
8755
-
"version": "1.0.0-rc.2-8feace6",
8756
-
"resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-1.0.0-rc.2-8feace6.tgz",
8757
-
"integrity": "sha512-5yLmJTezMNI1gUEeGf0TJAr05nK1BSs82w/L3wJNHFlnQXSFbMdxQXGcDAivGGQEuiBwZTl7gMgDNsrXXQnZIg==",
8755
+
"version": "1.0.0-rc.2-e38a2ba",
8756
+
"resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-1.0.0-rc.2-e38a2ba.tgz",
8757
+
"integrity": "sha512-m+S4nvYsgBhUPPjRWEPb9dybO2WcZXs+L6h2ziWOmFnjIgQ2jiDIKVQUuaCMGyKBFISxS6kjfMAJ1zzeQqWSFw==",
8758
8758
"license": "Apache-2.0",
8759
8759
"peerDependencies": {
8760
8760
"@aws-sdk/client-rds-data": ">=3",
+3
-3
package.json
+3
-3
package.json
···
85
85
"dbus-ts": "^0.0.7",
86
86
"discord.js": "^14.26.0",
87
87
"dotenv": "^10.0.0",
88
-
"drizzle-orm": "^1.0.0-rc.2-8feace6",
88
+
"drizzle-orm": "^1.0.0-rc.2-c5a84d1",
89
89
"express": "^5.2.1",
90
90
"express-session": "^1.19.0",
91
91
"fast-equals": "^6.0.0",
···
140
140
"devDependencies": {
141
141
"@chakra-ui/react": "^3.34.0",
142
142
"@chromatic-com/storybook": "^5.1.2",
143
-
"ansi-to-react": "^6.2.6",
144
143
"@dbus-types/notifications": "^0.0.5",
145
144
"@emotion/react": "^11.14.0",
146
145
"@eslint/js": "^10.0.1",
···
180
179
"@types/superagent": "^8.1.9",
181
180
"@types/xml2js": "^0.4.11",
182
181
"@vitejs/plugin-react": "^6.0.1",
182
+
"ansi-to-react": "^6.2.6",
183
183
"autoprefixer": "^10.4.27",
184
184
"chai": "^4.3.6",
185
185
"chai-as-promised": "^8.0.2",
186
186
"clsx": "^2.1.1",
187
-
"drizzle-kit": "^1.0.0-rc.1-929a083",
187
+
"drizzle-kit": "^1.0.0-rc.2-c5a84d1",
188
188
"eslint": "^10.3.0",
189
189
"eslint-plugin-prefer-arrow-functions": "^3.2.4",
190
190
"eslint-plugin-storybook": "^10.3.6",
+4
-4
src/backend/common/AbstractComponent.ts
+4
-4
src/backend/common/AbstractComponent.ts
···
47
47
protected transformManager: TransformerManager;
48
48
protected cache: MSCache;
49
49
protected db: DbConcrete;
50
-
protected componentRepo: DrizzleComponentRepository;
51
-
protected dbComponent: ComponentSelect;
50
+
protected componentRepo!: DrizzleComponentRepository;
51
+
protected dbComponent!: ComponentSelect;
52
52
protected retentionOpts: RetentionOptions;
53
53
54
54
protected componentType: 'source' | 'client';
···
58
58
super(config);
59
59
this.transformManager = config.transformManager ?? getRoot().items.transformerManager;
60
60
this.cache = getRoot().items.cache();
61
-
this.db = getRoot().items.db();
62
-
this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger});
63
61
const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES, {lower: true});
64
62
if(!cProps.every(isCompactableProperty)) {
65
63
throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`);
···
88
86
name = this.name as string;
89
87
}
90
88
89
+
this.db = await getRoot().items.db();
90
+
this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger});
91
91
this.dbComponent = await this.componentRepo.findOrInsert({
92
92
mode: this.componentType,
93
93
type: this.type,
+20
-21
src/backend/common/database/Database.ts
+20
-21
src/backend/common/database/Database.ts
···
10
10
import dayjs from 'dayjs';
11
11
import { parseDurationFromDurationValue } from '../../utils/TimeUtils.js';
12
12
import assert, { AssertionError } from 'node:assert';
13
+
import * as sqlite from 'node:sqlite';
13
14
import { parseBoolStrict } from '../../utils.js';
14
15
import { SimpleError } from '../errors/MSErrors.js';
15
16
···
23
24
return path.resolve(workingDirectory ?? configDir, `${name}.db`);
24
25
}
25
26
26
-
export const backupDb = async (dbName: string, opts: { logger?: Logger, workingDirectory?: string } = {}): Promise<void> => {
27
+
export const getDbBackupPath = (dbPath: string, suffix?: string): string => {
28
+
const pathInfo = path.parse(dbPath);
29
+
const backupPath = `${path.join(pathInfo.dir, pathInfo.name)}.db${suffix !== undefined ? `.${suffix}` : ''}.bak`;
30
+
return backupPath;
31
+
}
32
+
33
+
export const backupDb = async (db: sqlite.DatabaseSync, dbPath: string, opts: { logger?: Logger, suffix?: string } = {}): Promise<void | Uint8Array> => {
27
34
28
35
const {
29
36
logger: parentLogger = loggerNoop,
30
-
workingDirectory
37
+
suffix,
31
38
} = opts;
32
39
33
40
const logger = childLogger(parentLogger, 'Migrations');
34
41
35
-
const dbPath = getDbPath(dbName, workingDirectory);
36
-
let newDb = false;
42
+
if(dbPath === MEMORY_DB_NAME) {
43
+
// TODO serialize
44
+
return;
45
+
}
37
46
38
-
if (dbPath !== MEMORY_DB_NAME) {
39
-
if (!fileExists(dbPath)) {
40
-
logger.info(`Database at ${dbPath} does not exist, will create it.`);
41
-
newDb = true;
42
-
}
43
-
try {
44
-
fileOrDirectoryIsWriteable(dbPath);
45
-
} catch (e) {
46
-
throw new Error('Database path/folder is not writeable, cannot backup database', { cause: e });
47
+
const backupPath = getDbBackupPath(dbPath, suffix);
48
+
logger.info(`Backing up database to => ${backupPath}`);
49
+
await sqlite.backup(db, backupPath, {
50
+
progress: ({totalPages, remainingPages}) => {
51
+
logger.debug(`Backup in progress => ${totalPages - remainingPages} / ${totalPages}`);
47
52
}
48
-
}
49
-
50
-
if (dbPath !== MEMORY_DB_NAME && !newDb) {
51
-
const backupPath = `${getDbPath(`${Date.now()}-${dbName}`, workingDirectory)}.bak`;
52
-
logger.info(`Backing up database before migrating => ${backupPath}`);
53
-
await fs.copyFile(dbPath, backupPath)
54
-
logger.info('Backed up!');
55
-
}
53
+
})
54
+
logger.info('Backed up!');
56
55
}
57
56
58
57
const parseRetentionValue = (val: RetentionValueUnparsed): RetentionValue => {
+71
-24
src/backend/common/database/drizzle/drizzleUtils.ts
+71
-24
src/backend/common/database/drizzle/drizzleUtils.ts
···
4
4
import { sql as dsl, LogWriter, Logger as DrizzleLogger } from 'drizzle-orm';
5
5
import * as fs from 'fs/promises';
6
6
import * as path from 'path';
7
-
import { backupDb, getDbPath, MEMORY_DB_NAME } from '../Database.js';
8
-
import { fileExists } from '../../../utils/FSUtils.js';
7
+
import { backupDb, getDbPath, getDbBackupPath, MEMORY_DB_NAME } from '../Database.js';
8
+
import { fileExists, fileOrDirectoryIsWriteable } from '../../../utils/FSUtils.js';
9
9
import { childLogger, Logger, LogLevel } from '@foxxmd/logging';
10
10
import { loggerNoop } from '../../MaybeLogger.js';
11
11
import { projectDir } from '../../index.js';
12
12
import { relations } from './schema/schema.js';
13
13
import { addToContext, executeQuery } from './logContext.js';
14
+
import { DatabaseSync } from 'node:sqlite';
14
15
15
-
export async function shouldBackupDb(dbPath: string, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> {
16
+
export async function shouldBackupDb(dbVal: string | DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}): Promise<[boolean, string[]]> {
16
17
const {
17
18
logger: parentLogger = loggerNoop,
18
19
migrationsFolder = path.resolve(projectDir, 'src/backend/common/database/drizzle/migrations')
19
20
} = opts;
20
21
const logger = childLogger(parentLogger, 'Migrations');
21
22
22
-
logger.info(`Checking database at ${dbPath}`);
23
-
if (dbPath !== MEMORY_DB_NAME && !fileExists(dbPath)) {
24
-
logger.info(`No database exists!`);
25
-
return [false, []];
23
+
let db: DbConcrete;
24
+
25
+
if(typeof dbVal === 'string') {
26
+
logger.info(`Checking for database at ${dbVal}`);
27
+
if (dbVal !== MEMORY_DB_NAME && !fileExists(dbVal)) {
28
+
logger.info(`No database exists, no backup needed.`);
29
+
return [false, []];
30
+
}
31
+
db = await getDb(dbVal);
32
+
} else {
33
+
db = dbVal;
26
34
}
27
-
28
-
const db = drizzle(dbPath);
29
35
30
36
try {
31
37
// Ensure the migrations table exists
···
61
67
} catch (error) {
62
68
logger.error(new Error('Failed to get pending migrations', { cause: error }));
63
69
return [true, []];
64
-
} finally {
65
-
if(db.$client.isOpen) {
66
-
db.$client.close();
67
-
}
68
70
}
69
71
}
70
72
71
-
export const getDb = (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string } = {}) => {
73
+
// TODO backup path
74
+
export const getDb = (dbVal: string, opts: { logger?: Logger, backupPath?: string } = {}) => {
72
75
const {
73
-
workingDirectory,
74
76
logger = loggerNoop,
75
77
} = opts;
76
-
const dbPath = getDbPath(dbName, workingDirectory);
77
-
return drizzle(dbPath, {relations: relations, logger: createDrizzleLogger(logger)});
78
+
return drizzle(dbVal, {relations: relations, logger: createDrizzleLogger(logger)});
78
79
}
79
80
80
81
export type DbConcrete = ReturnType<typeof getDb>;
81
82
82
-
export const migrateDb = async (db: ReturnType<typeof drizzle>, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
83
+
export const migrateDb = async (db: DbConcrete, opts: {logger?: Logger, migrationsFolder?: string} = {}) => {
83
84
const {
84
85
migrationsFolder,
85
86
logger: parentLogger = loggerNoop
···
111
112
}
112
113
}
113
114
114
-
export const performDbMigrationWithBackup = async (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string } = {}) => {
115
-
const dbPath = getDbPath(dbName, opts.workingDirectory);
115
+
export const getMigratedDb = async (dbPath: string, opts: { logger?: Logger, migrationsFolder?: string, backupPath?: string } = {}): Promise<[DbConcrete, boolean]> => {
116
+
const {
117
+
logger = loggerNoop
118
+
} = opts;
119
+
let db: DbConcrete,
120
+
isNew = false,
121
+
hasPendingMigrations: boolean = true;
122
+
if (dbPath !== MEMORY_DB_NAME) {
123
+
try {
124
+
fileOrDirectoryIsWriteable(dbPath);
125
+
} catch (e) {
126
+
throw new Error('Database directory is not accessible', { cause: e });
127
+
}
128
+
129
+
const backupPath = getDbBackupPath(dbPath);
130
+
131
+
if(!fileExists(dbPath) && fileExists(backupPath)) {
132
+
logger.info(`Detected no database, making a copy of backup to use as new db. Backup file: ${backupPath}`);
133
+
await fs.copyFile(backupPath, dbPath);
134
+
}
135
+
if (fileExists(dbPath)) {
136
+
db = await getDb(dbPath, opts);
137
+
const [shouldBackup, pendingMigrations] = await shouldBackupDb(db, opts);
138
+
if (shouldBackup) {
139
+
hasPendingMigrations = true;
140
+
await backupDb(db.$client, dbPath, { logger: opts.logger });
141
+
}
142
+
} else {
143
+
logger.info('Detected no database, creating a new one...');
144
+
db = await getDb(dbPath, opts);
145
+
isNew = true;
146
+
}
147
+
} else {
148
+
logger.info('Detected in-memory database');
149
+
db = await getDb(dbPath, opts);
150
+
isNew = true;
151
+
}
116
152
117
-
const [shouldBackup, pendingMigrations] = await shouldBackupDb(dbPath, opts);
118
-
if(shouldBackup) {
119
-
await backupDb(dbName, opts);
153
+
if(hasPendingMigrations && dbPath !== MEMORY_DB_NAME) {
154
+
logger.info('TIP: Migrations may take some time, depending on the size of your database');
120
155
}
121
-
const db = getDb(dbName, opts);
122
156
await migrateDb(db, opts);
157
+
158
+
return [db, isNew];
123
159
}
160
+
161
+
// export const performDbMigrationWithBackup = async (dbName: string = 'ms', opts: { logger?: Logger, workingDirectory?: string, migrationsFolder?: string } = {}) => {
162
+
// const dbPath = getDbPath(dbName, opts.workingDirectory);
163
+
164
+
// const [shouldBackup, pendingMigrations] = await shouldBackupDb(dbPath, opts);
165
+
// if(shouldBackup) {
166
+
// await backupDb(dbName, opts);
167
+
// }
168
+
// const db = getDb(dbName, opts);
169
+
// await migrateDb(db, opts);
170
+
// }
124
171
125
172
export const createDrizzleLogger = (parentLogger: Logger, opts: {level?: LogLevel} = {}): DrizzleLogger => {
126
173
return {
+3
-3
src/backend/common/database/drizzle/repositories/BaseRepository.ts
+3
-3
src/backend/common/database/drizzle/repositories/BaseRepository.ts
···
1
1
import { childLogger, Logger } from "@foxxmd/logging";
2
-
import { getDb } from "../drizzleUtils.js";
2
+
import { DbConcrete } from "../drizzleUtils.js";
3
3
import { CompareOpKey } from "../drizzleTypes.js";
4
4
import { Dayjs } from "dayjs";
5
5
import { RelationsFieldFilter, eq, inArray } from "drizzle-orm";
···
41
41
displayName: string;
42
42
tableName: TableName;
43
43
table: ReturnType<typeof getConfigByTableName<T>>
44
-
db: ReturnType<typeof getDb>;
44
+
db: DbConcrete;
45
45
componentId?: number
46
46
47
-
constructor(db: ReturnType<typeof getDb>, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
47
+
constructor(db: DbConcrete, tableName: TableName, displayName: string, opts: DrizzleRepositoryOpts = {}) {
48
48
this.db = db;
49
49
this.displayName = displayName;
50
50
this.tableName = tableName;
+2
-2
src/backend/common/database/drizzle/repositories/ComponentRepository.ts
+2
-2
src/backend/common/database/drizzle/repositories/ComponentRepository.ts
···
1
1
import { Logger } from "drizzle-orm";
2
2
import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
3
-
import { getDb } from "../drizzleUtils.js";
3
+
import { DbConcrete } from "../drizzleUtils.js";
4
4
import { ComponentNew, ComponentSelect, FindWhere } from "../drizzleTypes.js";
5
5
import { components } from "../schema/schema.js";
6
6
import { generateComponentEntity } from "../entityUtils.js";
7
7
8
8
export class DrizzleComponentRepository extends DrizzleBaseRepository<'components'> {
9
9
10
-
constructor(db: ReturnType<typeof getDb>, opts: DrizzleRepositoryOpts = {}) {
10
+
constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
11
11
super(db, 'components', 'Component', opts);
12
12
}
13
13
+2
-2
src/backend/common/database/drizzle/repositories/PlayRepository.ts
+2
-2
src/backend/common/database/drizzle/repositories/PlayRepository.ts
···
1
1
import { childLogger, Logger, LoggerAppExtras } from "@foxxmd/logging";
2
-
import { DbConcrete, getDb, runTransaction } from "../drizzleUtils.js";
2
+
import { DbConcrete, runTransaction } from "../drizzleUtils.js";
3
3
import { loggerNoop } from "../../../MaybeLogger.js";
4
4
import { ErrorLike, PlayObject, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_EXACT, TemporalAccuracy } from "../../../../../core/Atomic.js";
5
5
import { generateInputEntity, generatePlayEntity, PlayEntityOpts, hydratePlaySelect, PlayHydateOptions } from "../entityUtils.js";
···
69
69
protected getQueueNextPrepared?: ReturnType<typeof this.prepareGetQueueNext>
70
70
protected getQueuedScrobbleRangePrepared?: ReturnType<typeof this.prepareGetQueuedScrobbleRange>
71
71
72
-
constructor(db: ReturnType<typeof getDb>, opts: DrizzleRepositoryOpts = {}) {
72
+
constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
73
73
super(db, 'plays', 'Plays', opts);
74
74
}
75
75
+2
-2
src/backend/common/database/drizzle/repositories/QueueRepository.ts
+2
-2
src/backend/common/database/drizzle/repositories/QueueRepository.ts
···
1
1
import { eq, and, lte, inArray } from "drizzle-orm";
2
2
import { DrizzleBaseRepository, DrizzleRepositoryOpts } from "./BaseRepository.js";
3
-
import { getDb } from "../drizzleUtils.js";
3
+
import { DbConcrete } from "../drizzleUtils.js";
4
4
import { QueueStateSelect } from "../drizzleTypes.js";
5
5
import { queueStates } from "../schema/schema.js";
6
6
import { CLIENT_DEAD_QUEUE } from "../../../../../core/Atomic.js";
7
7
export class DrizzleQueueRepository extends DrizzleBaseRepository<'queueStates'> {
8
8
9
-
constructor(db: ReturnType<typeof getDb>, opts: DrizzleRepositoryOpts = {}) {
9
+
constructor(db: DbConcrete, opts: DrizzleRepositoryOpts = {}) {
10
10
super(db, 'queueStates', 'Queue', opts);
11
11
}
12
12
+43
-4
src/backend/index.ts
+43
-4
src/backend/index.ts
···
21
21
import ScrobbleClients from './scrobblers/ScrobbleClients.js';
22
22
import ScrobbleSources from './sources/ScrobbleSources.js';
23
23
import { Notifiers } from './notifier/Notifiers.js';
24
-
import { getDb, performDbMigrationWithBackup } from './common/database/drizzle/drizzleUtils.js';
24
+
import { DbConcrete, getMigratedDb } from './common/database/drizzle/drizzleUtils.js';
25
25
import { getDbPath } from './common/database/Database.js';
26
26
import { createRetentionCleanupTask } from './tasks/retentionCleanup.js';
27
27
import { parseUserConfig } from './common/Cache.js';
···
51
51
52
52
let logger: FoxLogger;
53
53
54
+
let db: DbConcrete;
55
+
let dbConnectionsClosed = false;
56
+
54
57
process.on('uncaughtExceptionMonitor', (err, origin) => {
55
58
const appError = new Error(`Uncaught exception is crashing the app! :( Type: ${origin}`, {cause: err});
56
59
if(logger !== undefined) {
···
58
61
} else {
59
62
initLogger.error(appError);
60
63
}
64
+
if(!dbConnectionsClosed) {
65
+
const parts = [];
66
+
if(db !== undefined && !db.$client.isOpen) {
67
+
db.$client.close();
68
+
parts.push('Database');
69
+
}
70
+
if(parts.length > 0 && logger !== undefined) {
71
+
logger.info(`Closed ${parts.join(' and ')}`);
72
+
}
73
+
}
74
+
});
75
+
process.on('SIGINT', async () => {
76
+
if(!dbConnectionsClosed) {
77
+
const parts = [];
78
+
if(db !== undefined && !db.$client.isOpen) {
79
+
db.$client.close();
80
+
parts.push('Database');
81
+
}
82
+
if(parts.length > 0 && logger !== undefined) {
83
+
logger.info(`Closed ${parts.join(' and ')}`);
84
+
}
85
+
}
86
+
process.exit(0);
61
87
})
88
+
62
89
63
90
const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
64
91
···
98
125
const [aLogger, appLoggerStream] = await appLogger(logging)
99
126
logger = childLogger(aLogger, 'App');
100
127
101
-
logger.info(`Using database at ${getDbPath('ms')}`);
102
-
await performDbMigrationWithBackup('ms', {logger});
128
+
const dbPath = getDbPath('ms');
129
+
logger.info(`Using database at ${db}`);
130
+
const [migratedDb, isNew] = await getMigratedDb(dbPath, {logger: childLogger(logger, 'DB')});
131
+
db = migratedDb;
103
132
104
133
const root = getRoot({
105
134
...config,
···
107
136
logger,
108
137
loggingConfig: logging,
109
138
loggerStream: appLoggerStream,
110
-
db: getDb('ms', {logger})
139
+
db
111
140
});
112
141
113
142
const internalConfigOptional = {
···
222
251
logger.error(appError);
223
252
} else {
224
253
initLogger.error(appError);
254
+
}
255
+
if(!dbConnectionsClosed) {
256
+
const parts = [];
257
+
if(db !== undefined && !db.$client.isOpen) {
258
+
db.$client.close();
259
+
parts.push('Database');
260
+
}
261
+
if(parts.length > 0 && logger !== undefined) {
262
+
logger.info(`Closed ${parts.join(' and ')}`);
263
+
}
225
264
}
226
265
process.exit(1);
227
266
}
+4
-5
src/backend/ioc.ts
+4
-5
src/backend/ioc.ts
···
28
28
cache?: CacheConfigOptions | MSCache | (() => MSCache)
29
29
mbMap?: MusicBrainzSingletonMap | (() => MusicBrainzSingletonMap)
30
30
transformers?: TransformerCommonConfig[]
31
-
db?: DbConcrete | (() => DbConcrete)
31
+
db?: DbConcrete | (() => Promise<DbConcrete>)
32
32
}
33
33
34
34
const discovered = new prom.Counter({
···
92
92
maybeSingletonMb = new Map();
93
93
}
94
94
95
-
let dbFunc: () => DbConcrete;
96
-
let maybeSingletonDb: DbConcrete;
95
+
let dbFunc: () => Promise<DbConcrete>;
97
96
if(typeof db === 'function') {
98
97
dbFunc = db;
99
98
} else {
100
-
maybeSingletonDb = db;
99
+
dbFunc = async () => db;
101
100
}
102
101
103
102
const cEmitter = new WildcardEmitter();
···
158
157
cache: () => maybeSingletonCache !== undefined ? () => maybeSingletonCache : cacheFunc,
159
158
mbMap: () => maybeSingletonMb !== undefined ? () => maybeSingletonMb : mbFunc,
160
159
coverArtApi,
161
-
db: () => maybeSingletonDb !== undefined ? () => maybeSingletonDb : dbFunc
160
+
db: () => dbFunc
162
161
}).add((items) => {
163
162
const localUrl = generateBaseURL(baseUrl, items.port)
164
163
return {
+4
-4
src/backend/scrobblers/AbstractScrobbleClient.ts
+4
-4
src/backend/scrobblers/AbstractScrobbleClient.ts
···
152
152
153
153
declare protected componentType: 'client';
154
154
155
-
protected playRepo: DrizzlePlayRepository;
156
-
protected queueRepo: DrizzleQueueRepository;
155
+
protected playRepo!: DrizzlePlayRepository;
156
+
protected queueRepo!: DrizzleQueueRepository;
157
157
158
158
constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) {
159
159
super(config);
···
166
166
this.deadLogger = childLogger(this.logger, CLIENT_DEAD_QUEUE);
167
167
this.notifier = notifier;
168
168
this.emitter = emitter;
169
-
this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger,});
170
-
this.queueRepo = new DrizzleQueueRepository(this.db, {logger: this.logger});
171
169
172
170
const {
173
171
options: {
···
327
325
}
328
326
329
327
protected async postDatabase(): Promise<void> {
328
+
this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger});
329
+
this.queueRepo = new DrizzleQueueRepository(this.db, {logger: this.logger});
330
330
this.playRepo.componentId = this.dbComponent.id;
331
331
this.queueRepo.componentId = this.dbComponent.id;
332
332
this.tracksScrobbled = this.dbComponent.countLive + this.dbComponent.countNonLive;
+2
-2
src/backend/sources/AbstractSource.ts
+2
-2
src/backend/sources/AbstractSource.ts
···
109
109
110
110
declare protected componentType: 'source';
111
111
112
-
protected playRepo: DrizzlePlayRepository;
112
+
protected playRepo!: DrizzlePlayRepository;
113
113
114
114
existingDiscoveredPlay: (playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean) => Promise<PlayMatchResult>
115
115
···
130
130
this.emitter = emitter;
131
131
132
132
this.discoveredCounter = getRoot().items.sourceMetics.discovered;
133
-
this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger});
134
133
135
134
const existingScrobbleOpts: ExistingScrobbleOpts = {
136
135
logger: this.logger,
···
207
206
}
208
207
209
208
protected async postDatabase(): Promise<void> {
209
+
this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger});
210
210
this.tracksDiscovered = this.dbComponent.countLive;
211
211
this.playRepo.componentId = this.dbComponent.id;
212
212
}
+32
-43
src/backend/tests/database/drizzle.test.ts
+32
-43
src/backend/tests/database/drizzle.test.ts
···
1
1
import chai, { assert, expect } from 'chai';
2
2
import asPromised from 'chai-as-promised';
3
-
import { getDb, migrateDb, performDbMigrationWithBackup, shouldBackupDb } from '../../common/database/drizzle/drizzleUtils.js';
3
+
import { getDb, migrateDb, shouldBackupDb, getMigratedDb } from '../../common/database/drizzle/drizzleUtils.js';
4
4
import withLocalTmpDir from 'with-local-tmp-dir';
5
5
import { components, playInputs, plays, queueStates } from '../../common/database/drizzle/schema/schema.js';
6
6
import dayjs from 'dayjs';
···
10
10
import * as path from 'path';
11
11
import * as fs from 'fs/promises';
12
12
import { projectDir } from '../../common/index.js';
13
-
import { DatabaseSync } from 'node:sqlite';
14
13
import { fixtureCreateComponent, fixtureCreateInput, fixtureCreatePlay } from '../utils/databaseFixtures.js';
15
14
import { DrizzlePlayRepository, RepositoryCreatePlayOpts } from '../../common/database/drizzle/repositories/PlayRepository.js';
16
15
import { generatePlayWithLifecycle, generateRandomObj } from '../../../core/tests/utils/fixtures.js';
···
19
18
import { eq, sql } from 'drizzle-orm';
20
19
import { PlaySelect } from '../../common/database/drizzle/drizzleTypes.js';
21
20
import { loggerDebug } from '@foxxmd/logging';
21
+
import { transientDb } from '../utils/TransientTestUtils.js';
22
22
23
23
// would be great to push migrations directly from schema but doesn't seem supported in newest beta
24
24
// https://github.com/drizzle-team/drizzle-orm/discussions/4373
···
28
28
it('Detects non-existent db', async function () {
29
29
30
30
await withLocalTmpDir(async () => {
31
-
const [shouldBackup, pending] = await shouldBackupDb(getDbPath('notreal', process.cwd()));
32
-
expect(shouldBackup).is.false;
33
-
expect(pending).length(0);
31
+
const [db, isNew] = await getMigratedDb(getDbPath('notreal', process.cwd()));
32
+
expect(isNew).is.true;
33
+
db.$client.close();
34
34
}, {postfix: 'noDb'});
35
35
36
36
});
···
38
38
it('Detects abnormal db', async function () {
39
39
40
40
await withLocalTmpDir(async () => {
41
-
const otherdb = new DatabaseSync(path.resolve('./', 'other.db'));
42
-
const [shouldBackup, pending] = await shouldBackupDb(getDbPath('other', process.cwd()));
43
-
expect(shouldBackup).is.true;
44
-
expect(pending).length(0);
45
-
otherdb.close();
41
+
// database exists but there is no __drizzle_migrations table
42
+
const db = await getDb(':memory:');
43
+
const [shouldBackup, pending] = await shouldBackupDb(db);
44
+
expect(shouldBackup).is.true;
45
+
db.$client.close();
46
46
}, { unsafeCleanup: true, postfix: 'badDb' });
47
47
48
48
});
···
60
60
try {
61
61
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
62
62
const mf = path.resolve('./migrations');
63
-
const db = getDb('ms', { workingDirectory: process.cwd() });
63
+
const db = await getDb(':memory:');
64
64
await migrateDb(db, { migrationsFolder: mf });
65
65
const res = await x('drizzle-kit', [
66
66
'generate',
···
73
73
path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
74
74
'--dialect',
75
75
'sqlite'
76
-
]);
77
-
const [shouldBackup, pending] = await shouldBackupDb(getDbPath('ms', process.cwd()), { migrationsFolder: mf });
76
+
], {throwOnError: true});
77
+
const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf });
78
78
expect(shouldBackup).is.true;
79
79
expect(pending).length(1);
80
80
expect(pending[0]).includes('newMigration');
···
98
98
try {
99
99
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
100
100
const mf = path.resolve('./migrations');
101
-
const db = getDb('ms', { workingDirectory: process.cwd() });
101
+
const db = await getDb(':memory:');
102
102
await migrateDb(db, { migrationsFolder: mf });
103
-
const [shouldBackup, pending] = await shouldBackupDb(getDbPath('ms', process.cwd()), { migrationsFolder: mf });
103
+
const [shouldBackup, pending] = await shouldBackupDb(db, { migrationsFolder: mf });
104
104
expect(shouldBackup).is.false;
105
105
expect(pending).length(0);
106
106
db.$client.close();
···
118
118
119
119
await withLocalTmpDir(async () => {
120
120
121
+
const dbPath = getDbPath('msDb', process.cwd());
121
122
// copy first migration
122
123
await fs.mkdir('migrations');
123
124
try {
124
125
await fs.cp(path.resolve(projectDir, `src/backend/common/database/drizzle/migrations/${migrationFiles[0]}`), path.resolve('./migrations/', migrationFiles[0]), { recursive: true });
125
126
const mf = path.resolve('./migrations');
126
-
const db = getDb('ms', { workingDirectory: process.cwd() });
127
+
const [db, _] = await getMigratedDb(dbPath, {migrationsFolder: mf});
127
128
await migrateDb(db, { migrationsFolder: mf });
128
129
const res = await x('drizzle-kit', [
129
130
'generate',
···
136
137
path.resolve(projectDir, 'src/backend/common/database/drizzle/schema'),
137
138
'--dialect',
138
139
'sqlite'
139
-
]);
140
-
db.$client.close();
140
+
], {throwOnError: true});
141
141
142
142
// add dummy data to migration so migrate() doesn't fail
143
143
const newMigrationFolder = (await fs.readdir(path.resolve('./migrations/'))).find(x => x.includes('newMigration'));
144
144
await fs.appendFile(path.resolve('./migrations/',newMigrationFolder, 'migration.sql'),`\nselect count(*) from plays;`);
145
145
146
-
await performDbMigrationWithBackup('ms', {workingDirectory: process.cwd(), migrationsFolder: mf});
146
+
await getMigratedDb(dbPath, {migrationsFolder: mf});
147
147
const contents = await fs.readdir(path.resolve('./'));
148
148
expect(contents.some(x => x.includes('ms.db.bak')));
149
149
} catch (e) {
···
158
158
159
159
it('Should create a play', async function () {
160
160
161
-
const db = getDb(':memory:', { workingDirectory: process.cwd() });
161
+
const db = await transientDb();
162
162
await migrateDb(db);
163
163
164
164
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
···
177
177
178
178
it('Should create a play with relations', async function () {
179
179
180
-
const db = getDb(':memory:', { workingDirectory: process.cwd() });
180
+
const db = await transientDb();
181
181
await migrateDb(db);
182
182
183
183
try {
···
226
226
227
227
it('deletes all dependent relations when a Play is deleted', async function () {
228
228
229
-
const db = getDb(':memory:', { workingDirectory: process.cwd() });
229
+
const db = await transientDb();
230
230
await migrateDb(db);
231
231
232
232
try {
···
301
301
302
302
it('creates Plays and inputs', async function () {
303
303
304
-
const db = getDb(':memory:');
305
-
await migrateDb(db);
304
+
const db = await transientDb();
306
305
307
306
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
308
307
···
331
330
332
331
it('finds Plays by state', async function () {
333
332
334
-
const db = getDb(':memory:');
335
-
await migrateDb(db);
336
-
333
+
const db = await transientDb();
337
334
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
338
335
339
336
const repo = new DrizzlePlayRepository(db, {componentId: component[0].id});
···
361
358
362
359
it('finds Plays by date range', async function () {
363
360
364
-
const db = getDb(':memory:');
365
-
await migrateDb(db);
361
+
const db = await transientDb();
366
362
367
363
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
368
364
···
410
406
411
407
it('finds Plays by component', async function () {
412
408
413
-
const db = getDb(':memory:');
414
-
await migrateDb(db);
409
+
const db = await transientDb();
415
410
416
411
const component1 = await db.insert(components).values(fixtureCreateComponent()).returning();
417
412
const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning();
···
457
452
458
453
it('finds purgable Plays', async function () {
459
454
460
-
const db = getDb(':memory:');
461
-
await migrateDb(db);
455
+
const db = await transientDb();
462
456
463
457
const component1 = await db.insert(components).values(fixtureCreateComponent()).returning();
464
458
const component2 = await db.insert(components).values(fixtureCreateComponent({ uid: 'test2', name: 'jelly2' })).returning();
···
523
517
524
518
it('Get json property from play', async function () {
525
519
526
-
const db = getDb(':memory:', { workingDirectory: process.cwd() });
527
-
await migrateDb(db);
520
+
const db = await transientDb();
528
521
529
522
try {
530
523
···
565
558
566
559
await withLocalTmpDir(async () => {
567
560
try {
568
-
let db = getDb('ms', { workingDirectory: process.cwd() });
569
-
await migrateDb(db);
561
+
let [db, _] = await getMigratedDb(getDbPath('ms', process.cwd()));
570
562
const stats = await fs.stat(path.resolve('./ms.db'));
571
563
loggerDebug.debug(`Empty => ${stats.size / 1024}kb`);
572
564
} catch (e) {
···
581
573
582
574
await withLocalTmpDir(async () => {
583
575
try {
584
-
let db = getDb('ms', { workingDirectory: process.cwd() });
585
-
await migrateDb(db);
576
+
let [db, _] = await getMigratedDb(getDbPath('ms', process.cwd()));
586
577
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
587
578
588
579
const playRepo = new DrizzlePlayRepository(db);
···
615
606
616
607
await withLocalTmpDir(async () => {
617
608
try {
618
-
let db = getDb('ms', { workingDirectory: process.cwd() });
619
-
await migrateDb(db);
609
+
let [db, _] = await getMigratedDb(getDbPath('ms', process.cwd()));
620
610
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
621
611
622
612
const playRepo = new DrizzlePlayRepository(db);
···
649
639
650
640
await withLocalTmpDir(async () => {
651
641
try {
652
-
let db = getDb('ms', { workingDirectory: process.cwd() });
653
-
await migrateDb(db);
642
+
let [db, _] = await getMigratedDb(getDbPath('ms', process.cwd()));
654
643
const component = await db.insert(components).values(fixtureCreateComponent()).returning();
655
644
656
645
const playRepo = new DrizzlePlayRepository(db);
+6
-2
src/backend/tests/scrobbler/TestScrobbler.ts
+6
-2
src/backend/tests/scrobbler/TestScrobbler.ts
···
31
31
this.scrobbleDelay = 10;
32
32
this.scrobbleSleep = 20;
33
33
this.scrobbleWaitStopInterval = 20;
34
-
this.playRepoTest = this.playRepo;
35
-
this.queueRepoTest = this.queueRepo;
36
34
}
37
35
38
36
doScrobble(playObj: PlayObject) {
···
42
40
protected async doParseCache() {
43
41
await this.cache.init();
44
42
return super.doParseCache();
43
+
}
44
+
45
+
protected async postDatabase(): Promise<void> {
46
+
super.postDatabase();
47
+
this.playRepoTest = this.playRepo;
48
+
this.queueRepoTest = this.queueRepo;
45
49
}
46
50
47
51
playToClientPayload(playObject: PlayObject): object {
+10
-1
src/backend/tests/source/source.test.ts
+10
-1
src/backend/tests/source/source.test.ts
···
426
426
});
427
427
});
428
428
429
+
class DeezerTestSource extends DeezerInternalSource {
430
+
protected async doCheckConnection(): Promise<true | string | undefined> {
431
+
return;
432
+
}
433
+
doAuthentication = async () => {
434
+
return true;
435
+
}
436
+
}
437
+
429
438
const generateDeezerSource = async (options: DeezerInternalSourceOptions = {}) => {
430
-
const source = new DeezerInternalSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
439
+
const source = new DeezerTestSource('test', {data: {arl: 'test'}, options}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest, version: 'test'}, emitter);
431
440
await source.tryInitialize();
432
441
return source;
433
442
}
+1
-1
src/backend/tests/utils/TransientTestUtils.ts
+1
-1
src/backend/tests/utils/TransientTestUtils.ts
+3
-3
src/backend/tests/ytm/ytm.test.ts
+3
-3
src/backend/tests/ytm/ytm.test.ts
···
140
140
const prependedPlays = [newPlay, ...plays];
141
141
expect(source.parseRecentAgainstResponse(prependedPlays).plays).length(1);
142
142
143
-
await sleep(1000);
143
+
await sleep(50);
144
144
145
145
// YT returns outdated history
146
146
// should be detected as append since "removed" track in last position from previous history is seen again
···
148
148
expect(badAppend).to.deep.include({consistent: false, diffType: 'added', plays: []});
149
149
expect(badAppend.diffResults[2]).eq('append');
150
150
151
-
await sleep(500);
151
+
await sleep(10);
152
152
153
153
// contiuned outdated history
154
154
expect(source.parseRecentAgainstResponse(plays)).to.deep.include({consistent: true, plays: []});
155
155
156
-
await sleep(500);
156
+
await sleep(10);
157
157
158
158
// correct, current history is finally returned correctly
159
159
const recentHistoryResult = source.parseRecentAgainstResponse(prependedPlays);
+1
-1
src/core/StringUtils.ts
+1
-1
src/core/StringUtils.ts
···
230
230
/**
231
231
* Returns value if it is a non-empty string or returns default value
232
232
* */
233
-
export const nonEmptyStringOrDefault = <T>(str: any, defaultVal: T = undefined): string | T => {
233
+
export const nonEmptyStringOrDefault = <T = undefined>(str: any, defaultVal: T = undefined): string | T => {
234
234
if (str === undefined || str === null || typeof str !== 'string' || str.trim() === '') {
235
235
return defaultVal;
236
236
}
+4
-1
vite.config.ts
+4
-1
vite.config.ts
···
33
33
console.debug(`[VITE] BASE_URL ENV: ${process.env.BASE_URL} | Base Url String: ${baseUrlStr}`);
34
34
return {
35
35
server: {
36
-
allowedHosts: (true as true)
36
+
allowedHosts: (true as true),
37
+
watch: {
38
+
ignored: ['**/config/**']
39
+
}
37
40
},
38
41
esbuild: {
39
42
minifyIdentifiers: false