feat: Implement secret interpolation from ENV · foxxmd.dev/multi-scrobbler@36ed40f

+3 -2

src/backend/common/vendor/LastfmApiClient.ts

··· 7 7 } from "lastfm-node-client"; 8 8 import { PlayObject } from "../../../core/Atomic.js"; 9 9 import { nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js"; 10 - import { readJson, removeUndefinedKeys, sleep, writeFile } from "../../utils.js"; 10 + import { removeUndefinedKeys, sleep, writeFile } from "../../utils.js"; 11 + import { readJson } from '../../utils/DataUtils.js'; 11 12 import { joinedUrl } from "../../utils/NetworkUtils.js"; 12 13 import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js"; 13 14 import { getNodeNetworkException, isNodeNetworkException } from "../errors/NodeErrors.js"; ··· 169 170 initialize = async (): Promise<true> => { 170 171 171 172 try { 172 - const creds = await readJson(this.workingCredsPath, {throwOnNotFound: false}); 173 + const creds = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}); 173 174 const {sessionKey} = creds || {}; 174 175 if (this.client.sessionKey === undefined && sessionKey !== undefined) { 175 176 this.client.sessionKey = sessionKey;

+3 -2

src/backend/index.ts

··· 16 16 import { initServer } from "./server/index.js"; 17 17 import { createHeartbeatClientsTask } from "./tasks/heartbeatClients.js"; 18 18 import { createHeartbeatSourcesTask } from "./tasks/heartbeatSources.js"; 19 - import { isDebugMode, parseBool, readJson, retry, sleep } from "./utils.js"; 19 + import { isDebugMode, parseBool, retry, sleep } from "./utils.js"; 20 + import { readJson } from './utils/DataUtils.js'; 20 21 import { createVegaGenerator } from './utils/SchemaUtils.js'; 21 22 import ScrobbleClients from './scrobblers/ScrobbleClients.js'; 22 23 import ScrobbleSources from './sources/ScrobbleSources.js'; ··· 62 63 let appConfigFail: Error | undefined = undefined; 63 64 let config = {}; 64 65 try { 65 - config = await readJson(`${configDir}/config.json`, {throwOnNotFound: false}); 66 + config = await readJson(`${configDir}/config.json`, {throwOnNotFound: false, logger: childLogger(initLogger, 'Secrets')}); 66 67 } catch (e) { 67 68 appConfigFail = e; 68 69 }

+4 -3

src/backend/scrobblers/ScrobbleClients.ts

··· 10 10 import { MalojaClientConfig } from "../common/infrastructure/config/client/maloja.js"; 11 11 import { WildcardEmitter } from "../common/WildcardEmitter.js"; 12 12 import { Notifiers } from "../notifier/Notifiers.js"; 13 - import { isDebugMode, readJson } from "../utils.js"; 13 + import { isDebugMode } from "../utils.js"; 14 + import { readJson } from '../utils/DataUtils.js'; 14 15 import { joinedUrl } from "../utils/NetworkUtils.js"; 15 16 import { getTypeSchemaFromConfigGenerator } from "../utils/SchemaUtils.js"; 16 17 import { validateJson } from "../utils/ValidationUtils.js"; ··· 128 129 129 130 let configFile; 130 131 try { 131 - configFile = await readJson(`${this.configDir}/config.json`, {throwOnNotFound: false}); 132 + configFile = await readJson(`${this.configDir}/config.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `Secrets`)}); 132 133 } catch (e) { 133 134 // think this should stay as show-stopper since config could include important defaults (delay, retries) we don't want to ignore 134 135 throw new Error('config.json could not be parsed'); ··· 294 295 } 295 296 let rawClientConfigs; 296 297 try { 297 - rawClientConfigs = await readJson(`${this.configDir}/${clientType}.json`, {throwOnNotFound: false}); 298 + rawClientConfigs = await readJson(`${this.configDir}/${clientType}.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `${clientType} Secrets`)}); 298 299 } catch (e) { 299 300 const errMsg = `${clientType}.json config file could not be parsed`; 300 301 this.emitter.emit('error', errMsg);

+1 -1

src/backend/sources/DeezerInternalSource.ts

··· 5 5 import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; 6 6 import { DeezerInternalSourceConfig, DeezerInternalTrackData, DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; 7 7 import { TRANSFORM_HOOK } from "../common/infrastructure/Transform.js"; 8 - import { parseRetryAfterSecsFromObj, playObjDataMatch, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; 8 + import { parseRetryAfterSecsFromObj, playObjDataMatch, sleep, sortByOldestPlayDate} from "../utils.js"; 9 9 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; 10 10 import { CookieJar, Cookie } from 'tough-cookie'; 11 11 import { MixedCookieAgent } from 'http-cookie-agent/http';

+3 -2

src/backend/sources/DeezerSource.ts

··· 6 6 import { PlayObject } from "../../core/Atomic.js"; 7 7 import { DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; 8 8 import { DeezerSourceConfig } from "../common/infrastructure/config/source/deezer.js"; 9 - import { parseRetryAfterSecsFromObj, readJson, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; 9 + import { parseRetryAfterSecsFromObj, sleep, sortByOldestPlayDate, writeFile, } from "../utils.js"; 10 + import { readJson } from '../utils/DataUtils.js'; 10 11 import { joinedUrl } from "../utils/NetworkUtils.js"; 11 12 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; 12 13 ··· 95 96 this.logger.warn('This Source is DEPRECATED! Deezer has dropped support official API support. New apps cannot be created and existing apps are not guaranteed to continue working. Refer to the MS documentation for a new Deezer Source implementation.'); 96 97 97 98 try { 98 - const credFile = await readJson(this.workingCredsPath, {throwOnNotFound: false}); 99 + const credFile = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}); 99 100 if(credFile !== undefined) { 100 101 this.config.data.accessToken = credFile.accessToken; 101 102 } else {

+4 -3

src/backend/sources/ScrobbleSources.ts

··· 34 34 import { WebScrobblerSourceConfig } from "../common/infrastructure/config/source/webscrobbler.js"; 35 35 import { YTMusicData, YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; 36 36 import { WildcardEmitter } from "../common/WildcardEmitter.js"; 37 - import { parseBool, readJson } from "../utils.js"; 37 + import { parseBool } from "../utils.js"; 38 + import { readJson } from '../utils/DataUtils.js'; 38 39 import { validateJson } from "../utils/ValidationUtils.js"; 39 40 import AbstractSource from "./AbstractSource.js"; 40 41 import { AzuracastSource } from "./AzuracastSource.js"; ··· 270 271 271 272 let configFile; 272 273 try { 273 - configFile = await readJson(`${this.internalConfig.configDir}/config.json`, {throwOnNotFound: false}); 274 + configFile = await readJson(`${this.internalConfig.configDir}/config.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `Secrets`)}); 274 275 } catch (e) { 275 276 throw new Error('config.json could not be parsed'); 276 277 } ··· 711 712 } 712 713 let rawSourceConfigs; 713 714 try { 714 - rawSourceConfigs = await readJson(`${this.internalConfig.configDir}/${sourceType}.json`, {throwOnNotFound: false}); 715 + rawSourceConfigs = await readJson(`${this.internalConfig.configDir}/${sourceType}.json`, {throwOnNotFound: false, logger: childLogger(this.logger, `${sourceType} Secrets`)}); 715 716 } catch (e) { 716 717 const errMsg = `${sourceType}.json config file could not be parsed`; 717 718 this.emitter.emit('error', errMsg);

+2 -2

src/backend/sources/SpotifySource.ts

··· 19 19 import { SpotifySourceConfig } from "../common/infrastructure/config/source/spotify.js"; 20 20 import { 21 21 parseRetryAfterSecsFromObj, 22 - readJson, 23 22 sleep, 24 23 sortByOldestPlayDate, 25 24 writeFile, 26 25 } from "../utils.js"; 26 + import { readJson } from '../utils/DataUtils.js'; 27 27 import { findCauseByFunc } from "../utils/ErrorUtils.js"; 28 28 import { joinedUrl } from "../utils/NetworkUtils.js"; 29 29 import { RecentlyPlayedOptions } from "./AbstractSource.js"; ··· 219 219 220 220 let spotifyCreds = {}; 221 221 try { 222 - spotifyCreds = await readJson(this.workingCredsPath, {throwOnNotFound: false}) as any; 222 + spotifyCreds = await readJson(this.workingCredsPath, {throwOnNotFound: false, interpolateEnvs: false}) as any; 223 223 } catch (e) { 224 224 this.logger.warn('Current spotify credentials file exists but could not be parsed', { path: this.workingCredsPath }); 225 225 }

+1 -1

src/backend/tests/utils/strings.test.ts

··· 3 3 import { 4 4 compareNormalizedStrings, 5 5 normalizeStr, 6 - replaceInterpolatedValues, 7 6 } from "../../utils/StringUtils.js"; 7 + import { replaceInterpolatedValues } from "../../utils/DataUtils.js"; 8 8 import { splitByFirstFound } from '../../../core/StringUtils.js'; 9 9 import { noCasePropObj } from '../../utils/DataUtils.js'; 10 10

-19

src/backend/utils.ts

··· 5 5 import utc from 'dayjs/plugin/utc.js'; 6 6 import { Request } from "express"; 7 7 import { accessSync, constants, promises } from "fs"; 8 - import JSON5 from 'json5'; 9 8 // https://github.com/jfromaniello/url-join#in-nodejs 10 9 import pathUtil from "path"; 11 10 import { TimeoutError, WebapiError } from "spotify-web-api-node/src/response-error.js"; ··· 25 24 26 25 //const { default: Ajv } = AjvNS; 27 26 dayjs.extend(utc); 28 - 29 - export async function readJson(this: any, path: any, {throwOnNotFound = true} = {}) { 30 - try { 31 - await promises.access(path, constants.R_OK); 32 - const data = await promises.readFile(path); 33 - return JSON5.parse(data as unknown as string); 34 - } catch (e) { 35 - const {code} = e; 36 - if (code === 'ENOENT') { 37 - if (throwOnNotFound) { 38 - throw new Error(`No file found at given path: ${path}`, {cause: e}); 39 - } else { 40 - return; 41 - } 42 - } 43 - throw new Error(`Encountered error while parsing file: ${path}`, {cause: e}) 44 - } 45 - } 46 27 47 28 export async function readText(path: any) { 48 29 await promises.access(path, constants.R_OK);

+57 -1

src/backend/utils/DataUtils.ts

··· 1 + import JSON5 from "json5"; 2 + import { constants, promises } from "fs"; 3 + import { MaybeLogger } from "../common/logging.js"; 4 + 1 5 export const asArray = <T>(data: T | T[]): T[] => { 2 6 if (Array.isArray(data)) { 3 7 return data; ··· 82 86 newObj[i] = checkAtomic(obj[i]); 83 87 } 84 88 return newObj; // object with upper cased keys 85 - } 89 + } 90 + export async function readJson(this: any, path: any, options: ReadJsonOptions = {}) { 91 + const { 92 + throwOnNotFound = true, 93 + interpolateEnvs = true, 94 + logger 95 + } = options; 96 + 97 + try { 98 + await promises.access(path, constants.R_OK); 99 + const data = (await promises.readFile(path)).toString(); 100 + if(interpolateEnvs) { 101 + const replaced = replaceInterpolatedValues(data, process.env, logger); 102 + return JSON5.parse(replaced); 103 + } 104 + return data; 105 + } catch (e) { 106 + const { code } = e; 107 + if (code === 'ENOENT') { 108 + if (throwOnNotFound) { 109 + throw new Error(`No file found at given path: ${path}`, { cause: e }); 110 + } else { 111 + return; 112 + } 113 + } 114 + throw new Error(`Encountered error while parsing file: ${path}`, { cause: e }); 115 + } 116 + }export interface ReadJsonOptions { 117 + throwOnNotFound?: boolean; 118 + interpolateEnvs?: boolean; 119 + logger?: MaybeLogger 120 + } 121 + export const replaceInterpolatedValues = (str: string, fromVals: Record<string, any>, logger: MaybeLogger = new MaybeLogger()): string => { 122 + const cleanFromValKeys = noCasePropObj(fromVals); 123 + 124 + const matched = new Set(), unmatched = new Set(); 125 + const replaced = str.replaceAll(INTERPOLATION_WRAPPED_REGEX, (match, p1) => { 126 + //const fv = cleanFromValKeys[p1.toLocaleLowerCase().trim()]; 127 + const fv = cleanFromValKeys[p1.trim()]; 128 + if (fv !== undefined) { 129 + matched.add(p1); 130 + return fv; 131 + } 132 + unmatched.add(p1); 133 + return match; 134 + }); 135 + if (matched.size !== 0 || unmatched.size !== 0) { 136 + logger.debug(`Matched: ${matched.size === 0 ? 'None' : Array.from(matched.values()).join(', ')} | Unmatched: ${unmatched.size === 0 ? 'None' : Array.from(unmatched.values()).join(', ')}`); 137 + } 138 + return replaced; 139 + }; 140 + export const INTERPOLATION_WRAPPED_REGEX: RegExp = new RegExp(/\[\[([^\r\n\[\]]+?)\]\]/g); 141 +

-22

src/backend/utils/StringUtils.ts

··· 4 4 import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js"; 5 5 import { genGroupIdStr, getPlatformIdFromData, intersect, parseRegexSingleOrFail } from "../utils.js"; 6 6 import { buildTrackString } from "../../core/StringUtils.js"; 7 - import { MaybeLogger } from "../common/logging.js"; 8 - import { noCasePropObj } from "./DataUtils.js"; 9 7 10 8 const {levenStrategy, diceStrategy} = strategies; 11 9 ··· 434 432 return NON_ALPHANUMWHITESPACE_CHARS.test(str); 435 433 } 436 434 437 - export const INTERPOLATION_WRAPPED_REGEX: RegExp = new RegExp(/\[\[([^\r\n\[\]]+?)\]\]/g); 438 - export const replaceInterpolatedValues = (str: string, fromVals: Record<string, any>, logger: MaybeLogger = new MaybeLogger()): string => { 439 - const cleanFromValKeys = noCasePropObj(fromVals); 440 435 441 - const matched = new Set(), 442 - unmatched = new Set(); 443 - const replaced = str.replaceAll(INTERPOLATION_WRAPPED_REGEX, (match, p1) => { 444 - //const fv = cleanFromValKeys[p1.toLocaleLowerCase().trim()]; 445 - const fv = cleanFromValKeys[p1.trim()]; 446 - if(fv !== undefined) { 447 - matched.add(p1); 448 - return fv; 449 - } 450 - unmatched.add(p1); 451 - return match; 452 - }); 453 - if(matched.size !== 0 || unmatched.size !== 0) { 454 - logger.debug(`Matched: ${matched.size === 0 ? 'None' : Array.from(matched.values()).join(', ')} | Unmatched: ${unmatched.size === 0 ? 'None' : Array.from(unmatched.values()).join(', ')}`); 455 - } 456 - return replaced; 457 - }

Configure Feed

Configure Feed