feat(tealfm): Stream repo CAR to monitor download progress · foxxmd.dev/multi-scrobbler@18c520c

+20 -2

src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts

··· 16 16 import { UpstreamError } from "../../errors/UpstreamError.js"; 17 17 import { decodeTid, generateTID } from '@ewanc26/tid'; 18 18 import { Duration } from "dayjs/plugin/duration.js"; 19 + import { streamBodyProgress } from "../../../utils/NetworkUtils.js"; 19 20 20 21 export abstract class AbstractBlueSkyApiClient extends AbstractApiClient implements PagelessTimeRangeListens { 21 22 ··· 81 82 } 82 83 83 84 async getCAR() { 84 - // wish there was a way stream this... 85 - return await this.agent.com.atproto.sync.getRepo({did: this.agent.sessionManager.did}); 85 + const resp = await this.agent.sessionManager.fetchHandler(`/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(this.agent.sessionManager.did)}`, { 86 + method: 'GET', 87 + // @ts-expect-error 88 + duplex: 'half', 89 + redirect: 'follow', 90 + headers: { 91 + ...(Object.fromEntries(this.agent.headers.entries())), 92 + Accept: 'application/vnd.ipld.car', 93 + } 94 + }); 95 + if(resp.status !== 200) { 96 + const text = await resp.text(); 97 + throw new UpstreamError(`Failed to fetch repo CAR file. Response was ${resp.status} with response ${text}`, {responseBody: text}); 98 + } 99 + return await streamBodyProgress(resp, { 100 + logger: this.logger, 101 + chunkDefaultSize: 1024 * 1024 * 5, // report progress every 5 MB 102 + fileHint: 'repo CAR' 103 + }); 86 104 } 87 105 88 106 async getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise<PagelessTimeRangeListensResult> {

+7 -4

src/backend/scrobblers/TealfmScrobbler.ts

··· 13 13 import { Notifiers } from "../notifier/Notifiers.js"; 14 14 15 15 import AbstractScrobbleClient, { nowPlayingUpdateByPlayDuration, shouldClearNPStatus } from "./AbstractScrobbleClient.js"; 16 - import { TealClientConfig } from "../common/infrastructure/config/client/tealfm.js"; 16 + import { ScrobbleRecord, TealClientConfig } from "../common/infrastructure/config/client/tealfm.js"; 17 17 import { BlueSkyAppApiClient } from "../common/vendor/bluesky/BlueSkyAppApiClient.js"; 18 18 import { BlueSkyOauthApiClient } from "../common/vendor/bluesky/BlueSkyOauthApiClient.js"; 19 19 import { AbstractBlueSkyApiClient, nowPlayingExpirationDuration, playToRecord, playToStatusRecord, recordToPlay } from "../common/vendor/bluesky/AbstractBlueSkyApiClient.js"; ··· 178 178 } 179 179 180 180 protected async doHydrateHistoricalScrobbles(opts: {allowFailures?: boolean, signal?: AbortSignal } = {}) { 181 + const logger = childLogger(this.logger, ['Historical Plays']); 181 182 const { 182 183 allowFailures = false, 183 184 signal 184 185 } = opts; 185 186 let file: string; 186 187 try { 188 + logger.verbose('Fetching scrobbles from PDS...'); 187 189 file = await this.fetchCarToFile(); 188 190 signal?.throwIfAborted(); 189 191 } catch (e) { 190 - throw new Error('Failed to fetch CAR repo file', {cause: e}); 192 + throw new Error('Failed to fetch repo CAR', {cause: e}); 191 193 } 192 194 193 195 try { 194 - await this.parseScrobblesFromCar(file, 100, {allowFailures, logger: childLogger(this.logger, ['Historical Plays']), signal}); 196 + await this.parseScrobblesFromCar(file, 100, {allowFailures, logger: logger, signal}); 195 197 } catch (e) { 196 198 throw new Error('Failed to convert CAR without any error', {cause: e}); 197 199 } finally { ··· 200 202 } 201 203 202 204 async fetchCarToFile() { 205 + 203 206 const filename = path.resolve(this.configDir, `${this.getSafeExternalId()}-${dayjs().unix()}.car`); 204 - await fsPromise.writeFile(filename, Buffer.from(((await this.client.getCAR()).data))); 207 + await fsPromise.writeFile(filename, Buffer.from(((await this.client.getCAR())))); 205 208 return filename; 206 209 } 207 210

+20

src/backend/utils/DataUtils.ts

··· 203 203 name: nonEmptyStringOrDefault(process.env[`${prefix}_NAME`], undefined), 204 204 enable: e !== undefined ? parseBoolStrict(e) : undefined 205 205 }, false); 206 + } 207 + 208 + const byteSizes = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] 209 + 210 + /** 211 + * Convert bytes into human readable size 212 + * 213 + * @see https://stackoverflow.com/a/18650828/1469797 214 + */ 215 + export const formatBytes = (bytes: number, decimals: number = 2): [string, number, string] => { 216 + if (!+bytes) return ['0 Bytes', 0, 'Byes']; 217 + 218 + const k = 1024 219 + const dm = decimals < 0 ? 0 : decimals 220 + 221 + const i = Math.floor(Math.log(bytes) / Math.log(k)) 222 + 223 + const friendlySize = parseFloat((bytes / Math.pow(k, i)).toFixed(dm)); 224 + const friendlyUnit = byteSizes[i]; 225 + return [`${friendlySize} ${friendlyUnit}`, friendlySize, friendlyUnit]; 206 226 }

+78 -1

src/backend/utils/NetworkUtils.ts

··· 8 8 import { URLData } from "../../core/Atomic.js"; 9 9 import { CloseEvent, ErrorEvent, RetryEvent } from 'iso-websocket' 10 10 import { WEBSOCKET_CLOSE_CODE_REASONS } from "../common/infrastructure/Atomic.js"; 11 + import { loggerNoop } from "../common/MaybeLogger.js"; 12 + import { formatBytes } from "./DataUtils.js"; 11 13 12 14 export interface PortReachableOpts { 13 15 host: string, ··· 265 267 default: 266 268 return state.toString(); 267 269 } 268 - } 270 + } 271 + 272 + export type StreamBodyOpts = { 273 + logger?: Logger, 274 + chunkDefaultSize?: number, 275 + fileHint?: string 276 + } 277 + 278 + export const streamBodyProgress = async (response: Response, opts: StreamBodyOpts = {}) => { 279 + const { 280 + logger = loggerNoop, 281 + chunkDefaultSize = 1024 * 1024 * 10, // default to every 10MB, when we don't know response size 282 + fileHint = 'file' 283 + } = opts; 284 + let loading = true, 285 + chunks: any[] = []; 286 + const reader = response.body.getReader(); 287 + 288 + let length: number, 289 + chunkReportSize: number = chunkDefaultSize, 290 + lastReportedSize: number = 0; 291 + if(null !== response.headers.get('content-length')) { 292 + length = +response.headers.get('content-length'); 293 + const [summary, size, unit] = formatBytes(length); 294 + if(unit === 'MiB' && size > 10) { 295 + switch(true) { 296 + case(size > 50): 297 + chunkReportSize = length/10; 298 + break; 299 + case(size > 20): 300 + chunkReportSize = length/5; 301 + break; 302 + default: 303 + chunkReportSize = length/3; 304 + break; 305 + } 306 + } 307 + logger.trace(`Downloading ${summary} ${fileHint}...`); 308 + } else { 309 + logger.trace(`Downloading ${fileHint} of unknown size (no content-length header)...`); 310 + } 311 + 312 + let received = 0; 313 + 314 + // Loop through the response stream and extract data chunks 315 + while (loading) { 316 + const { done, value } = await reader.read(); 317 + if (done) { 318 + // Finish loading 319 + loading = false; 320 + } else { 321 + // Push values to the chunk array 322 + chunks.push(value); 323 + 324 + received += value.length; 325 + lastReportedSize += value.length; 326 + if(lastReportedSize >= chunkReportSize) { 327 + logger.trace(`Downloaded ${formatBytes(received)[0]}...`); 328 + lastReportedSize = 0; 329 + } 330 + } 331 + } 332 + logger.trace(`Finished download!`); 333 + 334 + // Concat the chunks into a single array 335 + let body = new Uint8Array(received); 336 + let position = 0; 337 + 338 + // Order the chunks by their respective position 339 + for (let chunk of chunks) { 340 + body.set(chunk, position); 341 + position += chunk.length; 342 + } 343 + 344 + return body; 345 + }

Configure Feed

Configure Feed