+241 -155

package-lock.json

Reviewed

··· 17 17 "@awaitjs/express": "^0.6.3", 18 18 "@curvenote/ansi-to-react": "^7.0.0", 19 19 "@donedeal0/superdiff": "^1.1.1", 20 20 + "@ewanc26/tid": "^1.0.2", 20 21 "@fortawesome/fontawesome-svg-core": "^6.4.2", 21 22 "@fortawesome/free-solid-svg-icons": "^6.4.2", 22 23 "@fortawesome/react-fontawesome": "^0.2.0", ··· 85 86 "ntfy": "^1.7.6", 86 87 "p-event": "^4.2.0", 87 88 "p-map": "^7.0.4", 89 89 + "p-retry": "^7.1.1", 88 90 "passport": "^0.6.0", 89 91 "passport-deezer": "^0.2.0", 90 92 "patch-package": "^8.0.0", ··· 139 141 "@types/react": "^18.2.18", 140 142 "@types/react-dom": "^18.2.7", 141 143 "@types/react-window": "^1.8.5", 144 144 + "@types/sinon": "^21.0.0", 142 145 "@types/spotify-web-api-node": "^5.0.7", 143 146 "@types/superagent": "^8.1.9", 144 147 "@types/xml2js": "^0.4.11", ··· 150 153 "git-cliff": "^2.12.0", 151 154 "mocha": "^10.3.0", 152 155 "mockdate": "^3.0.5", 153 153 - "msw": "^2.1.2", 156 156 + "msw": "^2.12.10", 154 157 "nodemon": "^3.0.3", 158 158 + "sinon": "^21.0.2", 155 159 "ts-essentials": "^9.1.2", 156 160 "typescript": "5.5.4", 157 161 "typescript-eslint": "^7.0.1", ··· 961 965 "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.2.1.tgz", 962 966 "integrity": "sha512-gdWzq7eX017a1kZCU/bP/sbk4e0GZ6idjsXOcMrQwODCb/rx985fHJJ8+hCu79KpuG7PfZh7bo3BBjPH37JuZw==" 963 967 }, 964 964 - "node_modules/@bundled-es-modules/cookie": { 965 965 - "version": "2.0.0", 966 966 - "resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.0.tgz", 967 967 - "integrity": "sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==", 968 968 - "dev": true, 969 969 - "dependencies": { 970 970 - "cookie": "^0.5.0" 971 971 - } 972 972 - }, 973 973 - "node_modules/@bundled-es-modules/cookie/node_modules/cookie": { 974 974 - "version": "0.5.0", 975 975 - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", 976 976 - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", 977 977 - "dev": true, 978 978 - "engines": { 979 979 - "node": ">= 0.6" 980 980 - } 981 981 - }, 982 982 - "node_modules/@bundled-es-modules/statuses": { 983 983 - "version": "1.0.1", 984 984 - "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz", 985 985 - "integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==", 986 986 - "dev": true, 987 987 - "dependencies": { 988 988 - "statuses": "^2.0.1" 989 989 - } 990 990 - }, 991 991 - "node_modules/@bundled-es-modules/tough-cookie": { 992 992 - "version": "0.1.6", 993 993 - "resolved": "https://registry.npmjs.org/@bundled-es-modules/tough-cookie/-/tough-cookie-0.1.6.tgz", 994 994 - "integrity": "sha512-dvMHbL464C0zI+Yqxbz6kZ5TOEp7GLW+pry/RWndAR8MJQAXZ2rPmIs8tziTZjeIyhSNZgZbCePtfSbdWqStJw==", 995 995 - "dev": true, 996 996 - "dependencies": { 997 997 - "@types/tough-cookie": "^4.0.5", 998 998 - "tough-cookie": "^4.1.4" 999 999 - } 1000 1000 - }, 1001 1001 - "node_modules/@bundled-es-modules/tough-cookie/node_modules/tough-cookie": { 1002 1002 - "version": "4.1.4", 1003 1003 - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", 1004 1004 - "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", 1005 1005 - "dev": true, 1006 1006 - "license": "BSD-3-Clause", 1007 1007 - "dependencies": { 1008 1008 - "psl": "^1.1.33", 1009 1009 - "punycode": "^2.1.1", 1010 1010 - "universalify": "^0.2.0", 1011 1011 - "url-parse": "^1.5.3" 1012 1012 - }, 1013 1013 - "engines": { 1014 1014 - "node": ">=6" 1015 1015 - } 1016 1016 - }, 1017 1017 - "node_modules/@bundled-es-modules/tough-cookie/node_modules/universalify": { 1018 1018 - "version": "0.2.0", 1019 1019 - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", 1020 1020 - "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", 1021 1021 - "dev": true, 1022 1022 - "license": "MIT", 1023 1023 - "engines": { 1024 1024 - "node": ">= 4.0.0" 1025 1025 - } 1026 1026 - }, 1027 968 "node_modules/@curvenote/ansi-to-react": { 1028 969 "version": "7.0.0", 1029 970 "resolved": "https://registry.npmjs.org/@curvenote/ansi-to-react/-/ansi-to-react-7.0.0.tgz", ··· 1714 1655 "engines": { 1715 1656 "node": "^12.22.0 || ^14.17.0 || >=16.0.0" 1716 1657 } 1658 1658 + }, 1659 1659 + "node_modules/@ewanc26/tid": { 1660 1660 + "version": "1.0.2", 1661 1661 + "resolved": "https://registry.npmjs.org/@ewanc26/tid/-/tid-1.0.2.tgz", 1662 1662 + "integrity": "sha512-d5zESrFEcEUSH3V2LomGwo1Lz+q0mUWyvkKS3jvbpeanQpRM4zM17ZauuCDzKMjJ+mFM6laVAMIpNgbVcyyPVA==", 1663 1663 + "license": "AGPL-3.0-only" 1717 1664 }, 1718 1665 "node_modules/@faker-js/faker": { 1719 1666 "version": "9.1.0", ··· 2301 2248 } 2302 2249 }, 2303 2250 "node_modules/@mswjs/interceptors": { 2304 2304 - "version": "0.36.7", 2305 2305 - "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.36.7.tgz", 2306 2306 - "integrity": "sha512-sdx02Wlus5hv6Bx7uUDb25gb0WGjCuSgnJB2LVERemoSGuqkZMe3QI6nEXhieFGtYwPrZbYrT2vPbsFN2XfbUw==", 2251 2251 + "version": "0.41.3", 2252 2252 + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.41.3.tgz", 2253 2253 + "integrity": "sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==", 2307 2254 "dev": true, 2255 2255 + "license": "MIT", 2308 2256 "dependencies": { 2309 2257 "@open-draft/deferred-promise": "^2.2.0", 2310 2258 "@open-draft/logger": "^0.3.0", ··· 2356 2304 "version": "2.2.0", 2357 2305 "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", 2358 2306 "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", 2359 2359 - "dev": true 2307 2307 + "dev": true, 2308 2308 + "license": "MIT" 2360 2309 }, 2361 2310 "node_modules/@open-draft/logger": { 2362 2311 "version": "0.3.0", 2363 2312 "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", 2364 2313 "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", 2365 2314 "dev": true, 2315 2315 + "license": "MIT", 2366 2316 "dependencies": { 2367 2317 "is-node-process": "^1.2.0", 2368 2318 "outvariant": "^1.4.0" ··· 2372 2322 "version": "2.1.0", 2373 2323 "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", 2374 2324 "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", 2375 2375 - "dev": true 2325 2325 + "dev": true, 2326 2326 + "license": "MIT" 2376 2327 }, 2377 2328 "node_modules/@opentelemetry/api": { 2378 2329 "version": "1.9.0", ··· 3032 2983 "url": "https://github.com/sponsors/sindresorhus" 3033 2984 } 3034 2985 }, 2986 2986 + "node_modules/@sinonjs/commons": { 2987 2987 + "version": "3.0.1", 2988 2988 + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", 2989 2989 + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", 2990 2990 + "dev": true, 2991 2991 + "license": "BSD-3-Clause", 2992 2992 + "dependencies": { 2993 2993 + "type-detect": "4.0.8" 2994 2994 + } 2995 2995 + }, 2996 2996 + "node_modules/@sinonjs/commons/node_modules/type-detect": { 2997 2997 + "version": "4.0.8", 2998 2998 + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", 2999 2999 + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", 3000 3000 + "dev": true, 3001 3001 + "license": "MIT", 3002 3002 + "engines": { 3003 3003 + "node": ">=4" 3004 3004 + } 3005 3005 + }, 3006 3006 + "node_modules/@sinonjs/fake-timers": { 3007 3007 + "version": "15.1.1", 3008 3008 + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.1.1.tgz", 3009 3009 + "integrity": "sha512-cO5W33JgAPbOh07tvZjUOJ7oWhtaqGHiZw+11DPbyqh2kHTBc3eF/CjJDeQ4205RLQsX6rxCuYOroFQwl7JDRw==", 3010 3010 + "dev": true, 3011 3011 + "license": "BSD-3-Clause", 3012 3012 + "dependencies": { 3013 3013 + "@sinonjs/commons": "^3.0.1" 3014 3014 + } 3015 3015 + }, 3016 3016 + "node_modules/@sinonjs/samsam": { 3017 3017 + "version": "9.0.2", 3018 3018 + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-9.0.2.tgz", 3019 3019 + "integrity": "sha512-H/JSxa4GNKZuuU41E3b8Y3tbSEx8y4uq4UH1C56ONQac16HblReJomIvv3Ud7ANQHQmkeSowY49Ij972e/pGxQ==", 3020 3020 + "dev": true, 3021 3021 + "license": "BSD-3-Clause", 3022 3022 + "dependencies": { 3023 3023 + "@sinonjs/commons": "^3.0.1", 3024 3024 + "type-detect": "^4.1.0" 3025 3025 + } 3026 3026 + }, 3035 3027 "node_modules/@standard-schema/spec": { 3036 3028 "version": "1.0.0", 3037 3029 "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.0.0.tgz", ··· 3782 3774 "@types/node": "*" 3783 3775 } 3784 3776 }, 3785 3785 - "node_modules/@types/cookie": { 3786 3786 - "version": "0.6.0", 3787 3787 - "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", 3788 3788 - "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", 3789 3789 - "dev": true 3790 3790 - }, 3791 3777 "node_modules/@types/cookiejar": { 3792 3778 "version": "2.1.5", 3793 3779 "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", ··· 4059 4045 "@types/send": "*" 4060 4046 } 4061 4047 }, 4048 4048 + "node_modules/@types/sinon": { 4049 4049 + "version": "21.0.0", 4050 4050 + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-21.0.0.tgz", 4051 4051 + "integrity": "sha512-+oHKZ0lTI+WVLxx1IbJDNmReQaIsQJjN2e7UUrJHEeByG7bFeKJYsv1E75JxTQ9QKJDp21bAa/0W2Xo4srsDnw==", 4052 4052 + "dev": true, 4053 4053 + "license": "MIT", 4054 4054 + "dependencies": { 4055 4055 + "@types/sinonjs__fake-timers": "*" 4056 4056 + } 4057 4057 + }, 4058 4058 + "node_modules/@types/sinonjs__fake-timers": { 4059 4059 + "version": "15.0.1", 4060 4060 + "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-15.0.1.tgz", 4061 4061 + "integrity": "sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==", 4062 4062 + "dev": true, 4063 4063 + "license": "MIT" 4064 4064 + }, 4062 4065 "node_modules/@types/spotify-api": { 4063 4066 "version": "0.0.25", 4064 4067 "resolved": "https://registry.npmjs.org/@types/spotify-api/-/spotify-api-0.0.25.tgz", ··· 4075 4078 } 4076 4079 }, 4077 4080 "node_modules/@types/statuses": { 4078 4078 - "version": "2.0.5", 4079 4079 - "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.5.tgz", 4080 4080 - "integrity": "sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==", 4081 4081 - "dev": true 4081 4081 + "version": "2.0.6", 4082 4082 + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.6.tgz", 4083 4083 + "integrity": "sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==", 4084 4084 + "dev": true, 4085 4085 + "license": "MIT" 4082 4086 }, 4083 4087 "node_modules/@types/superagent": { 4084 4088 "version": "8.1.9", ··· 4101 4105 "dependencies": { 4102 4106 "@types/jest": "*" 4103 4107 } 4104 4104 - }, 4105 4105 - "node_modules/@types/tough-cookie": { 4106 4106 - "version": "4.0.5", 4107 4107 - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", 4108 4108 - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", 4109 4109 - "dev": true 4110 4108 }, 4111 4109 "node_modules/@types/use-sync-external-store": { 4112 4110 "version": "0.0.3", ··· 7490 7488 "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==" 7491 7489 }, 7492 7490 "node_modules/graphql": { 7493 7493 - "version": "16.9.0", 7494 7494 - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.9.0.tgz", 7495 7495 - "integrity": "sha512-GGTKBX4SD7Wdb8mqeDLni2oaRGYQWjWHGKPQ24ZMnUtKfcsVoiv4uX8+LJr1K6U5VW2Lu1BwJnj7uiori0YtRw==", 7491 7491 + "version": "16.13.1", 7492 7492 + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.13.1.tgz", 7493 7493 + "integrity": "sha512-gGgrVCoDKlIZ8fIqXBBb0pPKqDgki0Z/FSKNiQzSGj2uEYHr1tq5wmBegGwJx6QB5S5cM0khSBpi/JFHMCvsmQ==", 7496 7494 "dev": true, 7495 7495 + "license": "MIT", 7497 7496 "engines": { 7498 7497 "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" 7499 7498 } ··· 8095 8094 "version": "1.1.0", 8096 8095 "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", 8097 8096 "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", 8098 8098 - "dev": true, 8099 8097 "license": "MIT", 8100 8098 "engines": { 8101 8099 "node": ">=16" ··· 8108 8106 "version": "1.2.0", 8109 8107 "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", 8110 8108 "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", 8111 8111 - "dev": true 8109 8109 + "dev": true, 8110 8110 + "license": "MIT" 8112 8111 }, 8113 8112 "node_modules/is-number": { 8114 8113 "version": "7.0.0", ··· 9642 9641 "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 9643 9642 }, 9644 9643 "node_modules/msw": { 9645 9645 - "version": "2.6.0", 9646 9646 - "resolved": "https://registry.npmjs.org/msw/-/msw-2.6.0.tgz", 9647 9647 - "integrity": "sha512-n3tx2w0MZ3H4pxY0ozrQ4sNPzK/dGtlr2cIIyuEsgq2Bhy4wvcW6ZH2w/gXM9+MEUY6HC1fWhqtcXDxVZr5Jxw==", 9644 9644 + "version": "2.12.10", 9645 9645 + "resolved": "https://registry.npmjs.org/msw/-/msw-2.12.10.tgz", 9646 9646 + "integrity": "sha512-G3VUymSE0/iegFnuipujpwyTM2GuZAKXNeerUSrG2+Eg391wW63xFs5ixWsK9MWzr1AGoSkYGmyAzNgbR3+urw==", 9648 9647 "dev": true, 9649 9648 "hasInstallScript": true, 9649 9649 + "license": "MIT", 9650 9650 "dependencies": { 9651 9651 - "@bundled-es-modules/cookie": "^2.0.0", 9652 9652 - "@bundled-es-modules/statuses": "^1.0.1", 9653 9653 - "@bundled-es-modules/tough-cookie": "^0.1.6", 9654 9651 "@inquirer/confirm": "^5.0.0", 9655 9655 - "@mswjs/interceptors": "^0.36.5", 9652 9652 + "@mswjs/interceptors": "^0.41.2", 9656 9653 "@open-draft/deferred-promise": "^2.2.0", 9657 9657 - "@open-draft/until": "^2.1.0", 9658 9658 - "@types/cookie": "^0.6.0", 9659 9659 - "@types/statuses": "^2.0.4", 9660 9660 - "chalk": "^4.1.2", 9661 9661 - "graphql": "^16.8.1", 9654 9654 + "@types/statuses": "^2.0.6", 9655 9655 + "cookie": "^1.0.2", 9656 9656 + "graphql": "^16.12.0", 9662 9657 "headers-polyfill": "^4.0.2", 9663 9658 "is-node-process": "^1.2.0", 9664 9659 "outvariant": "^1.4.3", 9665 9660 "path-to-regexp": "^6.3.0", 9661 9661 + "picocolors": "^1.1.1", 9662 9662 + "rettime": "^0.10.1", 9663 9663 + "statuses": "^2.0.2", 9666 9664 "strict-event-emitter": "^0.5.1", 9667 9667 - "type-fest": "^4.26.1", 9665 9665 + "tough-cookie": "^6.0.0", 9666 9666 + "type-fest": "^5.2.0", 9667 9667 + "until-async": "^3.0.2", 9668 9668 "yargs": "^17.7.2" 9669 9669 }, 9670 9670 "bin": { ··· 9699 9699 "node": ">=12" 9700 9700 } 9701 9701 }, 9702 9702 + "node_modules/msw/node_modules/cookie": { 9703 9703 + "version": "1.1.1", 9704 9704 + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", 9705 9705 + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", 9706 9706 + "dev": true, 9707 9707 + "license": "MIT", 9708 9708 + "engines": { 9709 9709 + "node": ">=18" 9710 9710 + }, 9711 9711 + "funding": { 9712 9712 + "type": "opencollective", 9713 9713 + "url": "https://opencollective.com/express" 9714 9714 + } 9715 9715 + }, 9702 9716 "node_modules/msw/node_modules/emoji-regex": { 9703 9717 "version": "8.0.0", 9704 9718 "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", ··· 9711 9725 "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", 9712 9726 "dev": true 9713 9727 }, 9728 9728 + "node_modules/msw/node_modules/statuses": { 9729 9729 + "version": "2.0.2", 9730 9730 + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", 9731 9731 + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", 9732 9732 + "dev": true, 9733 9733 + "license": "MIT", 9734 9734 + "engines": { 9735 9735 + "node": ">= 0.8" 9736 9736 + } 9737 9737 + }, 9714 9738 "node_modules/msw/node_modules/string-width": { 9715 9739 "version": "4.2.3", 9716 9740 "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", ··· 9725 9749 "node": ">=8" 9726 9750 } 9727 9751 }, 9752 9752 + "node_modules/msw/node_modules/tldts": { 9753 9753 + "version": "7.0.25", 9754 9754 + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.25.tgz", 9755 9755 + "integrity": "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w==", 9756 9756 + "dev": true, 9757 9757 + "license": "MIT", 9758 9758 + "dependencies": { 9759 9759 + "tldts-core": "^7.0.25" 9760 9760 + }, 9761 9761 + "bin": { 9762 9762 + "tldts": "bin/cli.js" 9763 9763 + } 9764 9764 + }, 9765 9765 + "node_modules/msw/node_modules/tldts-core": { 9766 9766 + "version": "7.0.25", 9767 9767 + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.25.tgz", 9768 9768 + "integrity": "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw==", 9769 9769 + "dev": true, 9770 9770 + "license": "MIT" 9771 9771 + }, 9772 9772 + "node_modules/msw/node_modules/tough-cookie": { 9773 9773 + "version": "6.0.0", 9774 9774 + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", 9775 9775 + "integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==", 9776 9776 + "dev": true, 9777 9777 + "license": "BSD-3-Clause", 9778 9778 + "dependencies": { 9779 9779 + "tldts": "^7.0.5" 9780 9780 + }, 9781 9781 + "engines": { 9782 9782 + "node": ">=16" 9783 9783 + } 9784 9784 + }, 9728 9785 "node_modules/msw/node_modules/type-fest": { 9729 9729 - "version": "4.26.1", 9730 9730 - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz", 9731 9731 - "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==", 9786 9786 + "version": "5.4.4", 9787 9787 + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.4.4.tgz", 9788 9788 + "integrity": "sha512-JnTrzGu+zPV3aXIUhnyWJj4z/wigMsdYajGLIYakqyOW1nPllzXEJee0QQbHj+CTIQtXGlAjuK0UY+2xTyjVAw==", 9732 9789 "dev": true, 9790 9790 + "license": "(MIT OR CC0-1.0)", 9791 9791 + "dependencies": { 9792 9792 + "tagged-tag": "^1.0.0" 9793 9793 + }, 9733 9794 "engines": { 9734 9734 - "node": ">=16" 9795 9795 + "node": ">=20" 9735 9796 }, 9736 9797 "funding": { 9737 9798 "url": "https://github.com/sponsors/sindresorhus" ··· 10484 10545 "version": "1.4.3", 10485 10546 "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", 10486 10547 "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", 10487 10487 - "dev": true 10548 10548 + "dev": true, 10549 10549 + "license": "MIT" 10488 10550 }, 10489 10551 "node_modules/p-cancelable": { 10490 10552 "version": "2.1.1", ··· 10559 10621 } 10560 10622 }, 10561 10623 "node_modules/p-retry": { 10562 10562 - "version": "6.2.1", 10563 10563 - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", 10564 10564 - "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", 10565 10565 - "dev": true, 10624 10624 + "version": "7.1.1", 10625 10625 + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", 10626 10626 + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", 10566 10627 "license": "MIT", 10567 10628 "dependencies": { 10568 10568 - "@types/retry": "0.12.2", 10569 10569 - "is-network-error": "^1.0.0", 10570 10570 - "retry": "^0.13.1" 10629 10629 + "is-network-error": "^1.1.0" 10571 10630 }, 10572 10631 "engines": { 10573 10573 - "node": ">=16.17" 10632 10632 + "node": ">=20" 10574 10633 }, 10575 10634 "funding": { 10576 10635 "url": "https://github.com/sponsors/sindresorhus" ··· 11261 11320 "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", 11262 11321 "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" 11263 11322 }, 11264 11264 - "node_modules/psl": { 11265 11265 - "version": "1.15.0", 11266 11266 - "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", 11267 11267 - "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", 11268 11268 - "dev": true, 11269 11269 - "license": "MIT", 11270 11270 - "dependencies": { 11271 11271 - "punycode": "^2.3.1" 11272 11272 - }, 11273 11273 - "funding": { 11274 11274 - "url": "https://github.com/sponsors/lupomontero" 11275 11275 - } 11276 11276 - }, 11277 11323 "node_modules/pstree.remy": { 11278 11324 "version": "1.1.8", 11279 11325 "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", ··· 11310 11356 "funding": { 11311 11357 "url": "https://github.com/sponsors/ljharb" 11312 11358 } 11313 11313 - }, 11314 11314 - "node_modules/querystringify": { 11315 11315 - "version": "2.2.0", 11316 11316 - "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", 11317 11317 - "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", 11318 11318 - "dev": true, 11319 11319 - "license": "MIT" 11320 11359 }, 11321 11360 "node_modules/queue-microtask": { 11322 11361 "version": "1.2.3", ··· 11702 11741 "dev": true, 11703 11742 "peer": true 11704 11743 }, 11705 11705 - "node_modules/requires-port": { 11706 11706 - "version": "1.0.0", 11707 11707 - "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", 11708 11708 - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", 11709 11709 - "dev": true, 11710 11710 - "license": "MIT" 11711 11711 - }, 11712 11744 "node_modules/reselect": { 11713 11745 "version": "4.1.8", 11714 11746 "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.8.tgz", ··· 11746 11778 "version": "0.13.1", 11747 11779 "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", 11748 11780 "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", 11781 11781 + "license": "MIT", 11749 11782 "engines": { 11750 11783 "node": ">= 4" 11751 11784 } 11785 11785 + }, 11786 11786 + "node_modules/rettime": { 11787 11787 + "version": "0.10.1", 11788 11788 + "resolved": "https://registry.npmjs.org/rettime/-/rettime-0.10.1.tgz", 11789 11789 + "integrity": "sha512-uyDrIlUEH37cinabq0AX4QbgV4HbFZ/gqoiunWQ1UqBtRvTTytwhNYjE++pO/MjPTZL5KQCf2bEoJ/BJNVQ5Kw==", 11790 11790 + "dev": true, 11791 11791 + "license": "MIT" 11752 11792 }, 11753 11793 "node_modules/reusify": { 11754 11794 "version": "1.0.4", ··· 12275 12315 "node": ">=10" 12276 12316 } 12277 12317 }, 12318 12318 + "node_modules/sinon": { 12319 12319 + "version": "21.0.2", 12320 12320 + "resolved": "https://registry.npmjs.org/sinon/-/sinon-21.0.2.tgz", 12321 12321 + "integrity": "sha512-VHV4UaoxIe5jrMd89Y9duI76T5g3Lp+ET+ctLhLDaZtSznDPah1KKpRElbdBV4RwqWSw2vadFiVs9Del7MbVeQ==", 12322 12322 + "dev": true, 12323 12323 + "license": "BSD-3-Clause", 12324 12324 + "dependencies": { 12325 12325 + "@sinonjs/commons": "^3.0.1", 12326 12326 + "@sinonjs/fake-timers": "^15.1.1", 12327 12327 + "@sinonjs/samsam": "^9.0.2", 12328 12328 + "diff": "^8.0.3", 12329 12329 + "supports-color": "^7.2.0" 12330 12330 + }, 12331 12331 + "funding": { 12332 12332 + "type": "opencollective", 12333 12333 + "url": "https://opencollective.com/sinon" 12334 12334 + } 12335 12335 + }, 12336 12336 + "node_modules/sinon/node_modules/diff": { 12337 12337 + "version": "8.0.3", 12338 12338 + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.3.tgz", 12339 12339 + "integrity": "sha512-qejHi7bcSD4hQAZE0tNAawRK1ZtafHDmMTMkrrIGgSLl7hTnQHmKCeB45xAcbfTqK2zowkM3j3bHt/4b/ARbYQ==", 12340 12340 + "dev": true, 12341 12341 + "license": "BSD-3-Clause", 12342 12342 + "engines": { 12343 12343 + "node": ">=0.3.1" 12344 12344 + } 12345 12345 + }, 12278 12346 "node_modules/slash": { 12279 12347 "version": "2.0.0", 12280 12348 "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", ··· 12411 12479 "version": "0.5.1", 12412 12480 "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", 12413 12481 "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", 12414 12414 - "dev": true 12482 12482 + "dev": true, 12483 12483 + "license": "MIT" 12415 12484 }, 12416 12485 "node_modules/string_decoder": { 12417 12486 "version": "1.3.0", ··· 13256 13325 "node": ">= 0.8" 13257 13326 } 13258 13327 }, 13328 13328 + "node_modules/until-async": { 13329 13329 + "version": "3.0.2", 13330 13330 + "resolved": "https://registry.npmjs.org/until-async/-/until-async-3.0.2.tgz", 13331 13331 + "integrity": "sha512-IiSk4HlzAMqTUseHHe3VhIGyuFmN90zMTpD3Z3y8jeQbzLIq500MVM7Jq2vUAnTKAFPJrqwkzr6PoTcPhGcOiw==", 13332 13332 + "dev": true, 13333 13333 + "license": "MIT", 13334 13334 + "funding": { 13335 13335 + "url": "https://github.com/sponsors/kettanaito" 13336 13336 + } 13337 13337 + }, 13259 13338 "node_modules/unws": { 13260 13339 "version": "0.2.4", 13261 13340 "resolved": "https://registry.npmjs.org/unws/-/unws-0.2.4.tgz", ··· 13303 13382 "dev": true, 13304 13383 "dependencies": { 13305 13384 "punycode": "^2.1.0" 13306 13306 - } 13307 13307 - }, 13308 13308 - "node_modules/url-parse": { 13309 13309 - "version": "1.5.10", 13310 13310 - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", 13311 13311 - "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", 13312 13312 - "dev": true, 13313 13313 - "license": "MIT", 13314 13314 - "dependencies": { 13315 13315 - "querystringify": "^2.1.1", 13316 13316 - "requires-port": "^1.0.0" 13317 13385 } 13318 13386 }, 13319 13387 "node_modules/use-sync-external-store": { ··· 13921 13989 }, 13922 13990 "funding": { 13923 13991 "url": "https://github.com/sponsors/dword-design" 13992 13992 + } 13993 13993 + }, 13994 13994 + "node_modules/with-local-tmp-dir/node_modules/p-retry": { 13995 13995 + "version": "6.2.1", 13996 13996 + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", 13997 13997 + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", 13998 13998 + "dev": true, 13999 13999 + "license": "MIT", 14000 14000 + "dependencies": { 14001 14001 + "@types/retry": "0.12.2", 14002 14002 + "is-network-error": "^1.0.0", 14003 14003 + "retry": "^0.13.1" 14004 14004 + }, 14005 14005 + "engines": { 14006 14006 + "node": ">=16.17" 14007 14007 + }, 14008 14008 + "funding": { 14009 14009 + "url": "https://github.com/sponsors/sindresorhus" 13924 14010 } 13925 14011 }, 13926 14012 "node_modules/word-wrap": {

+5 -1

package.json

Reviewed

··· 52 52 "@awaitjs/express": "^0.6.3", 53 53 "@curvenote/ansi-to-react": "^7.0.0", 54 54 "@donedeal0/superdiff": "^1.1.1", 55 55 + "@ewanc26/tid": "^1.0.2", 55 56 "@fortawesome/fontawesome-svg-core": "^6.4.2", 56 57 "@fortawesome/free-solid-svg-icons": "^6.4.2", 57 58 "@fortawesome/react-fontawesome": "^0.2.0", ··· 120 121 "ntfy": "^1.7.6", 121 122 "p-event": "^4.2.0", 122 123 "p-map": "^7.0.4", 124 124 + "p-retry": "^7.1.1", 123 125 "passport": "^0.6.0", 124 126 "passport-deezer": "^0.2.0", 125 127 "patch-package": "^8.0.0", ··· 174 176 "@types/react": "^18.2.18", 175 177 "@types/react-dom": "^18.2.7", 176 178 "@types/react-window": "^1.8.5", 179 179 + "@types/sinon": "^21.0.0", 177 180 "@types/spotify-web-api-node": "^5.0.7", 178 181 "@types/superagent": "^8.1.9", 179 182 "@types/xml2js": "^0.4.11", ··· 185 188 "git-cliff": "^2.12.0", 186 189 "mocha": "^10.3.0", 187 190 "mockdate": "^3.0.5", 188 188 - "msw": "^2.1.2", 191 191 + "msw": "^2.12.10", 189 192 "nodemon": "^3.0.3", 193 193 + "sinon": "^21.0.2", 190 194 "ts-essentials": "^9.1.2", 191 195 "typescript": "5.5.4", 192 196 "typescript-eslint": "^7.0.1",

+6 -4

src/backend/common/Cache.ts

Reviewed

··· 17 17 import { asCacheAuthProvider, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js'; 18 18 import { Typeson } from 'typeson'; 19 19 import { builtin } from 'typeson-registry'; 20 20 - import { MaybeLogger } from './logging.js'; 20 20 + import { loggerNoop } from './logging.js'; 21 21 import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js'; 22 22 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); 23 23 ··· 50 50 cacheAuth: Cacheable; 51 51 regexCache: ReturnType<typeof cacheFunctions>; 52 52 cacheTransform: Cacheable; 53 53 + cacheClientScrobbles: Cacheable; 53 54 54 55 logger: Logger; 55 56 ··· 96 97 97 98 this.regexCache = cacheFunctions(this.config.regex); 98 99 this.cacheTransform = new Cacheable({primary: initMemoryCache({lruSize: 500})}); 100 100 + this.cacheClientScrobbles = new Cacheable({primary: initMemoryCache({lruSize: 100, ttl: '5m'})}); 99 101 } 100 102 101 103 init = async () => { ··· 213 215 }); 214 216 } 215 217 216 216 - export const flatCacheLoad = async (flatCache: FlatCache, logger: MaybeLogger): Promise<void> => { 218 218 + export const flatCacheLoad = async (flatCache: FlatCache, logger: Logger = loggerNoop): Promise<void> => { 217 219 218 220 const cachePath = path.join(flatCache.cacheDir, flatCache.cacheId); 219 221 try { ··· 224 226 225 227 const streamPromise = new Promise((resolve, reject) => { 226 228 flatCache.loadFileStream(cachePath, (progress: number, total: number) => { 227 227 - logger.debug(`Loading ${progress}/${total} chunks...`); 229 229 + logger.trace(`Loading ${progress}/${total} chunks...`); 228 230 }, () => { 229 231 resolve(true); 230 232 }, (err: Error) => { ··· 260 262 } 261 263 } 262 264 263 263 - export const initFileCache = async (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => { 265 265 + export const initFileCache = async (opts: FlatCacheOptions = {}, logger: Logger = loggerNoop): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => { 264 266 const flatCache = flatCacheCreate(opts); 265 267 try { 266 268 await flatCacheLoad(flatCache, logger);

+89 -3

src/backend/common/infrastructure/Atomic.ts

Reviewed

··· 1 1 import { Logger } from '@foxxmd/logging'; 2 2 - import { Dayjs } from "dayjs"; 2 2 + import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; 3 3 + import { Dayjs, ManipulateType } from "dayjs"; 3 4 import { Request, Response } from "express"; 4 5 import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core"; 5 6 import { FixedSizeList } from 'fixed-size-list'; 6 6 - import { isPlayObject, PlayMeta, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless } from "../../../core/Atomic.js"; 7 7 + import { isPlayObject, PlayMeta, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js"; 7 8 import TupleMap from "../TupleMap.js"; 8 9 import { MusicBrainzApi } from 'musicbrainz-api'; 9 10 ··· 346 347 testAuth: () => Promise<any> 347 348 } 348 349 350 350 + export interface ScrobbleRangeFetchable { 351 351 + getScrobblesForTimeRange: TimeRangeListensFetcher 352 352 + } 353 353 + 349 354 export interface MdnsDeviceInfo { 350 355 name: string 351 356 type: string ··· 406 411 407 412 export type MusicBrainzSingletonMap = Map<string,MusicBrainzApi>; 408 413 414 414 + 415 415 + 409 416 /* https://websocket.org/reference/close-codes/ */ 410 417 export const WEBSOCKET_CLOSE_CODE_REASONS = { 411 418 1000: 'Closed gracefully', ··· 428 435 InternalError: 1011 429 436 } 430 437 431 431 - export const WEBSOCKET_CLOSE_CODES_RETRY = [1006,1011,1001]; 438 438 + export const WEBSOCKET_CLOSE_CODES_RETRY = [1006,1011,1001];export interface PaginatedLimit { 439 439 + /** per page max number of results to return */ 440 440 + limit?: number 441 441 + } 442 442 + 443 443 + export interface PaginatedTimeRangeOptions { 444 444 + /** Unix timestamp */ 445 445 + from: UnixTimestamp 446 446 + /** Unix timestamp */ 447 447 + to: UnixTimestamp 448 448 + /** maximum number of results to fetch before returning early */ 449 449 + fetchMax?: number 450 450 + } 451 451 + 452 452 + export type PaginatedTimeRangeCommonOptions = Partial<PaginatedTimeRangeOptions> & PaginatedLimit; 453 453 + 454 454 + export type CursorType = number | string; 455 455 + 456 456 + export interface PaginatedListensOptions<T extends CursorType> extends PaginatedLimit { 457 457 + cursor: T 458 458 + } 459 459 + 460 460 + export interface PagelessListensTimeRangeOptions extends PaginatedTimeRangeCommonOptions { 461 461 + } 462 462 + 463 463 + export interface PaginatedListensTimeRangeOptions<T extends CursorType = CursorType> extends Partial<PaginatedTimeRangeOptions>, PaginatedListensOptions<T> { 464 464 + } 465 465 + 466 466 + export interface PaginatedResults<T extends CursorType = CursorType> { 467 467 + total?: number 468 468 + more?: boolean 469 469 + cursorNext?: T 470 470 + order?: 'desc' | 'asc' 471 471 + } 472 472 + 473 473 + export interface PaginatedListens { 474 474 + getPaginatedListens(params: PaginatedListensOptions<CursorType>): Promise<{data: PlayObject[], meta: PaginatedListensOptions<CursorType> & PaginatedResults<CursorType>}> 475 475 + } 476 476 + 477 477 + export const hasPaginagedListens = (obj: Object): obj is PaginatedListens => { 478 478 + return 'getPaginatedListens' in obj; 479 479 + } 480 480 + 481 481 + export interface PaginatedTimeRangeListensResult<T extends CursorType> { 482 482 + data: PlayObject[]; 483 483 + meta: PaginatedListensTimeRangeOptions<T> & PaginatedResults<T>; 484 484 + } 485 485 + export interface PaginatedTimeRangeListens<T extends CursorType = CursorType> { 486 486 + getPaginatedTimeRangeListens(params: PaginatedListensTimeRangeOptions<T>): Promise<PaginatedTimeRangeListensResult<T>> 487 487 + getPaginatedUnitOfTime(): ManipulateType; 488 488 + } 489 489 + 490 490 + export const hasPaginatedTimeRangeListens = (obj: Object): obj is PaginatedTimeRangeListens => { 491 491 + return 'getPaginatedTimeRangeListens' in obj; 492 492 + } 493 493 + 494 494 + export interface PagelessTimeRangeListensResult { 495 495 + data: PlayObject[] 496 496 + meta: PagelessListensTimeRangeOptions & PaginatedResults 497 497 + } 498 498 + export interface PagelessTimeRangeListens { 499 499 + getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise<PagelessTimeRangeListensResult> 500 500 + getPaginatedUnitOfTime(): ManipulateType; 501 501 + } 502 502 + 503 503 + export const hasPagelessTimeRangeListens = (obj: Object): obj is PagelessTimeRangeListens => { 504 504 + return 'getPagelessTimeRangeListens' in obj; 505 505 + } 506 506 + 507 507 + export type PaginatedTimeRangeSource = PaginatedTimeRangeListens | PagelessTimeRangeListens; 508 508 + export type PaginatedSource = PaginatedListens | PaginatedTimeRangeSource; 509 509 + 510 510 + export type TimeRangeListensFetcher<T extends CursorType = CursorType> = (opts: PaginatedTimeRangeCommonOptions | PaginatedListensTimeRangeOptions<T>) => Promise<PlayObject[]> 511 511 + 512 512 + export interface ScrobbleRangeResult { 513 513 + plays: PlayObject[] 514 514 + fetchedAt: Dayjs 515 515 + } 516 516 + 517 517 + export const REFRESH_STALE_DEFAULT = 60;

+13

src/backend/common/infrastructure/config/client/koito.ts

Reviewed

··· 1 1 + import { UnixTimestamp } from "../../../../../core/Atomic.js" 1 2 import { RequestRetryOptions } from "../common.js" 2 3 import { CommonClientConfig, CommonClientData } from "./index.js" 3 4 ··· 13 14 /** ISO8601 timestamp */ 14 15 time: string 15 16 track: TrackResponse 17 17 + } 18 18 + 19 19 + export interface GetListensOptions { 20 20 + limit?: number 21 21 + page?: number 22 22 + week?: number 23 23 + month?: number 24 24 + year?: number 25 25 + 26 26 + // new in 0.1.0 27 27 + to?: UnixTimestamp 28 28 + from?: UnixTimestamp 16 29 } 17 30 18 31 export interface TrackResponse {

+8

src/backend/common/infrastructure/config/client/listenbrainz.ts

Reviewed

··· 38 38 export interface ListenBrainzClientAIOConfig extends ListenBrainzClientConfig { 39 39 type: 'listenbrainz' 40 40 } 41 41 + 42 42 + 43 43 + /** https://github.com/metabrainz/listenbrainz-server/pull/2572 44 44 + * https://github.com/metabrainz/listenbrainz-server/blob/master/listenbrainz/webserver/views/api_tools.py#L48 45 45 + */ 46 46 + export const MAX_ITEMS_PER_GET_LZ = 1000; 47 47 + export const DEFAULT_ITEMS_PER_GET_LZ = 25; 48 48 + export const DEFAULT_MS_ITEMS_PER_GET_LZ = 100;

+16

src/backend/common/logging.ts

Reviewed

··· 115 115 } 116 116 } 117 117 } 118 118 + 119 119 + const noopLog = (_: any, ...rest: any) => undefined; 120 120 + 121 121 + export const loggerNoop: Logger = { 122 122 + trace: noopLog, 123 123 + debug: noopLog, 124 124 + log: noopLog, 125 125 + info: noopLog, 126 126 + verbose: noopLog, 127 127 + warn: noopLog, 128 128 + error: noopLog, 129 129 + fatal: noopLog, 130 130 + silent: noopLog, 131 131 + level: 'silent', 132 132 + child: (_: any, ...rest: any) => loggerNoop as Logger 133 133 + } as unknown as Logger;

+131 -53

src/backend/common/vendor/LastfmApiClient.ts

Reviewed

··· 1 1 - import dayjs, { Dayjs } from "dayjs"; 2 2 - import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../core/Atomic.js"; 1 1 + import dayjs, { Dayjs, ManipulateType } from "dayjs"; 2 2 + import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData, Writeable } from "../../../core/Atomic.js"; 3 3 import { nonEmptyStringOrDefault, splitByFirstFound } from "../../../core/StringUtils.js"; 4 4 import { removeUndefinedKeys, sleep, writeFile } from "../../utils.js"; 5 5 import { objectIsEmpty, readJson } from '../../utils/DataUtils.js'; ··· 7 7 import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js"; 8 8 import { getNodeNetworkException, isNodeNetworkException } from "../errors/NodeErrors.js"; 9 9 import { UpstreamError } from "../errors/UpstreamError.js"; 10 10 - import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfigOptional } from "../infrastructure/Atomic.js"; 10 10 + import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, InternalConfigOptional, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult } from "../infrastructure/Atomic.js"; 11 11 import { LastfmData } from "../infrastructure/config/client/lastfm.js"; 12 12 import AbstractApiClient from "./AbstractApiClient.js"; 13 13 import { normalizeStr, parseArtistCredits } from "../../utils/StringUtils.js"; 14 14 - import { LastFMUser, LastFMAuth, LastFMTrack, LastFMUserGetRecentTracksResponse, LastFMBooleanNumber, LastFMUpdateNowPlayingResponse, LastFMUserGetInfoResponse } from 'lastfm-ts-api'; 14 14 + import { LastFMUser, LastFMAuth, LastFMTrack, LastFMUserGetRecentTracksResponse, LastFMBooleanNumber, LastFMUpdateNowPlayingResponse, LastFMUserGetInfoResponse, LastFMUserGetRecentTracksParams } from 'lastfm-ts-api'; 15 15 import clone from 'clone'; 16 16 import { IncomingMessage } from "http"; 17 17 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.js"; ··· 37 37 export const LASTFM_HOST = 'ws.audioscrobbler.com'; 38 38 export const LASTFM_PATH = '/2.0'; 39 39 40 40 - export default class LastfmApiClient extends AbstractApiClient { 40 40 + export default class LastfmApiClient extends AbstractApiClient implements PaginatedTimeRangeListens<number> { 41 41 42 42 user?: string; 43 43 declare config: LastfmData; ··· 243 243 } 244 244 } 245 245 246 246 - getRecentTracks = async (options: TracksFetchOptions = {}): Promise<PlayObject[]> => { 246 246 + getRecentTracksWithPagination = async (options: TracksFetchOptions = {}) => { 247 247 + const { page = 1, limit = 200, from, to } = options; 247 248 248 248 - let resp: LastFMUserGetRecentTracksResponse; 249 249 - try { 250 250 - resp = await this.callApi<LastFMUserGetRecentTracksResponse>(() => this.userApi.getRecentTracks({ 251 251 - ...options, 252 252 - user: this.user, 253 253 - // sk is required if user has "Hide recent listening" enabled on their profile 254 254 - sk: this.sessionKey, 255 255 - extended: 1 256 256 - })); 249 249 + return await this.getRecentTracks({limit, from, to, page, extended: 1}); 250 250 + } 257 251 258 258 - const { 259 259 - recenttracks: { 260 260 - track: list = [], 261 261 - } = {} 262 262 - } = resp; 263 263 - return list.reduce((acc: any, x: any) => { 264 264 - try { 265 265 - const formatted = formatPlayObj(x); 266 266 - const { 267 267 - data: { 268 268 - track, 269 269 - playDate, 270 270 - }, 271 271 - meta: { 272 272 - mbid, 273 273 - nowPlaying, 274 274 - } 275 275 - } = formatted; 276 276 - if (nowPlaying === true) { 277 277 - // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing 278 278 - // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp 279 279 - // so we'll just ignore it in the context of recent tracks since really we only want "tracks that have already finished being played" anyway 280 280 - this.logger.debug( { track, mbid }, `Ignoring 'now playing' track returned from ${this.upstreamName} client`); 281 281 - return acc; 282 282 - } else if (playDate === undefined) { 252 252 + getPaginatedUnitOfTime(): ManipulateType { 253 253 + return 'second'; 254 254 + } 255 255 + 256 256 + getPaginatedTimeRangeListens = async (fetchOptions: PaginatedListensTimeRangeOptions<number>, options: { includeNowPlaying?: boolean } = {}): Promise<PaginatedTimeRangeListensResult<number>> => { 257 257 + 258 258 + const resp = await this.getRecentTracksWithPagination(fetchOptions); 259 259 + 260 260 + const { includeNowPlaying = false } = options; 261 261 + 262 262 + const { 263 263 + recenttracks: { 264 264 + track: list = [], 265 265 + '@attr': { 266 266 + total, 267 267 + totalPages, 268 268 + page 269 269 + } 270 270 + } = {}, 271 271 + } = resp; 272 272 + 273 273 + const plays = list.reduce((acc: any, x: any) => { 274 274 + try { 275 275 + const formatted = formatPlayObj(x); 276 276 + const { 277 277 + data: { 278 278 + track, 279 279 + playDate, 280 280 + }, 281 281 + meta: { 282 282 + mbid, 283 283 + nowPlaying, 284 284 + } 285 285 + } = formatted; 286 286 + if (nowPlaying === true && !includeNowPlaying) { 287 287 + // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing 288 288 + // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp 289 289 + // so we'll just ignore it in the context of recent tracks since really we only want "tracks that have already finished being played" anyway 290 290 + this.logger.trace({ track, mbid }, `Ignoring 'now playing' track returned from ${this.upstreamName} client`); 291 291 + return acc; 292 292 + } else if (playDate === undefined) { 293 293 + if(nowPlaying === true) { 294 294 + formatted.data.playDate = dayjs(); 295 295 + } else { 283 296 this.logger.warn({ track, mbid }, `${this.upstreamName} recently scrobbled track did not contain a timestamp, omitting from time frame check`); 284 297 return acc; 285 298 } 286 286 - return acc.concat(formatted); 287 287 - } catch (e) { 288 288 - this.logger.warn(new Error(`Failed to format ${this.upstreamName} recently scrobbled track, omitting from time frame check`, { cause: e })); 289 289 - this.logger.debug('Full api response object:'); 290 290 - this.logger.debug(x); 291 291 - return acc; 299 299 + } 300 300 + return acc.concat(formatted); 301 301 + } catch (e) { 302 302 + this.logger.warn(new Error(`Failed to format ${this.upstreamName} recently scrobbled track, omitting from time frame check`, { cause: e })); 303 303 + this.logger.debug({ data: x }, 'Full api response object:'); 304 304 + return acc; 305 305 + } 306 306 + }, []); 307 307 + 308 308 + return { data: plays, meta: { ...fetchOptions, total: parseInt(total, 10), more: fetchOptions.cursor < parseInt(totalPages, 10) } }; 309 309 + 310 310 + } 311 311 + 312 312 + getRecentTracks = async (options: TracksFetchOptions = {}): Promise<LastFMUserGetRecentTracksResponse> => { 313 313 + 314 314 + const { to, from, extended = 1, ...rest } = options; 315 315 + 316 316 + const requestOpts: Writeable<LastFMUserGetRecentTracksParams> = { 317 317 + ...rest, 318 318 + // sk is required if user has "Hide recent listening" enabled on their profile 319 319 + sk: this.sessionKey, 320 320 + user: this.user, 321 321 + extended 322 322 + }; 323 323 + 324 324 + if (to !== undefined) { 325 325 + requestOpts.to = to.toString(); 326 326 + } 327 327 + if (from !== undefined) { 328 328 + requestOpts.from = from.toString(); 329 329 + } 330 330 + 331 331 + try { 332 332 + const resp = await this.callApi<LastFMUserGetRecentTracksResponse>(() => this.userApi.getRecentTracks(requestOpts)); 333 333 + const { 334 334 + recenttracks: { 335 335 + track = [], 336 336 + ...rest 337 337 + } = {}, 338 338 + ...restResp 339 339 + } = resp; 340 340 + const correctedResp: Writeable<LastFMUserGetRecentTracksResponse> = { 341 341 + ...restResp, 342 342 + // @ts-expect-error 343 343 + recenttracks: { 344 344 + ...rest, 345 345 + track: [], 292 346 } 293 293 - }, []); 347 347 + } 348 348 + // when the lfm api response only returns one scrobble it formats `track` as a singular scrobble object INSTEAD of an array 349 349 + // and in all other cases it should return an array of scrobble objects 350 350 + if (!Array.isArray(track)) { 351 351 + if (typeof track === 'object' && track !== null) { 352 352 + correctedResp.recenttracks.track = [track]; 353 353 + this.logger.debug('Converted `track` property to an array'); 354 354 + } else { 355 355 + this.logger.warn({ track }, `Expected track to be an array or object but it was ${track === null ? 'null' : typeof track}`); 356 356 + } 357 357 + } else { 358 358 + correctedResp.recenttracks.track = track; 359 359 + } 360 360 + return correctedResp; 294 361 } catch (e) { 295 295 - if(e.message.includes('Invalid resource specified')) { 362 362 + if (e.message.includes('Invalid resource specified')) { 296 363 // likely the user does not have any scrobbles on their profile yet 297 364 // https://github.com/FoxxMD/multi-scrobbler/issues/401#issuecomment-3749489057 298 365 // https://github.com/libre-fm/libre-fm/discussions/91#discussioncomment-15456070 299 366 // so we log as a warning and return empty array instead 300 300 - this.logger.warn(new Error('This error occurs when a librefm (and lastfm?) account has no existing scrobbles yet. If you are seeing this warning and this is not the case, please create an issue', {cause: e})); 301 301 - return []; 367 367 + this.logger.warn(new Error('This error occurs when a librefm (and lastfm?) account has no existing scrobbles yet. If you are seeing this warning and this is not the case, please create an issue', { cause: e })); 368 368 + return { 369 369 + recenttracks: { 370 370 + track: [], 371 371 + '@attr': { 372 372 + user: this.user, 373 373 + totalPages: '0', 374 374 + total: '0', 375 375 + page: '1', 376 376 + perPage: '50' 377 377 + } 378 378 + } 379 379 + } 302 380 } 303 303 - this.logger.debug(resp); 304 381 throw e; 305 382 } 306 383 } ··· 400 477 } 401 478 402 479 export interface TracksFetchOptions { 403 403 - to?: string 404 404 - from?: string 480 480 + to?: UnixTimestamp 481 481 + from?: UnixTimestamp 405 482 extended?: 0 | 1 406 483 page?: number 484 484 + /** Max 200. Default is 50 */ 407 485 limit?: number 408 486 } 409 487

+119 -82

src/backend/common/vendor/ListenbrainzApiClient.ts

Reviewed

··· 1 1 import { stringSameness } from '@foxxmd/string-sameness'; 2 2 import dayjs from "dayjs"; 3 3 import request, { Request, Response } from 'superagent'; 4 4 - import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../core/Atomic.js"; 4 4 + import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp, URLData } from "../../../core/Atomic.js"; 5 5 import { combinePartsToString, slice } from "../../../core/StringUtils.js"; 6 6 import { 7 7 findDelimiters, ··· 14 14 } from "../../utils/StringUtils.js"; 15 15 import { getScrobbleTsSOCDate } from "../../utils/TimeUtils.js"; 16 16 import { UpstreamError } from "../errors/UpstreamError.js"; 17 17 - import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions } from "../infrastructure/Atomic.js"; 18 18 - import { ListenBrainzClientData } from "../infrastructure/config/client/listenbrainz.js"; 17 17 + import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, DELIMITERS, FormatPlayObjectOptions, PagelessListensTimeRangeOptions, PagelessTimeRangeListens, PagelessTimeRangeListensResult } from "../infrastructure/Atomic.js"; 18 18 + import { DEFAULT_ITEMS_PER_GET_LZ, ListenBrainzClientData, MAX_ITEMS_PER_GET_LZ } from "../infrastructure/config/client/listenbrainz.js"; 19 19 import AbstractApiClient from "./AbstractApiClient.js"; 20 20 import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from '../../utils/NetworkUtils.js'; 21 21 import { isEmptyArrayOrUndefined, removeUndefinedKeys, unique } from '../../utils.js'; 22 22 - import {ListensResponse as KoitoListensResponse} from '../infrastructure/config/client/koito.js' 23 23 - import { listenObjectResponseToPlay } from './koito/KoitoApiClient.js'; 24 22 import { version } from '../../ioc.js'; 25 23 import { ListenPayload, ListenResponse, ListenType, MinimumTrack, SubmitListenAdditionalTrackInfo, SubmitPayload } from './listenbrainz/interfaces.js'; 26 24 import { baseFormatPlayObj } from '../../utils/PlayTransformUtils.js'; 27 27 - import { ScrobbleSubmitError } from '../errors/MSErrors.js'; 25 25 + import { ScrobbleSubmitError, SimpleError } from '../errors/MSErrors.js'; 26 26 + import pRetry from 'p-retry'; 27 27 + import { findCauseByFunc } from '../../utils/ErrorUtils.js'; 28 28 + import { isSuperAgentResponseError } from '../errors/ErrorUtils.js'; 29 29 + 28 30 29 31 interface SubmitOptions { 30 32 log?: boolean ··· 34 36 export interface ListensResponse { 35 37 count: number; 36 38 listens: ListenResponse[]; 39 39 + latest_listen_ts: number; 40 40 + oldest_listen_ts: number; 37 41 } 38 42 39 39 - export class ListenbrainzApiClient extends AbstractApiClient { 43 43 + export interface UserListensOptions { 44 44 + /** unix epoch timestamp, listens with listened_at less than (but not including) this value will be returned. */ 45 45 + max_ts?: UnixTimestamp 46 46 + /** unix epoch timestamp, listens with listened_at greater than (but not including) this value will be returned. */ 47 47 + min_ts?: UnixTimestamp 48 48 + /** number of listens to return. Max is `MAX_ITEMS_PER_GET_LZ` 49 49 + * 50 50 + * @default 25 51 51 + */ 52 52 + count?: number 53 53 + } 54 54 + 55 55 + export class ListenbrainzApiClient extends AbstractApiClient implements PagelessTimeRangeListens { 40 56 41 57 declare config: ListenBrainzClientData; 42 58 url: URLData; 43 43 - isKoito: boolean = false; 44 59 45 60 constructor(name: any, config: ListenBrainzClientData, options: AbstractApiOptions) { 46 61 super('ListenBrainz', name, config, options); ··· 58 73 this.logger.verbose(`Config URL: '${url ?? '(None Given)'}' => Normalized: '${this.url.url}'`) 59 74 } 60 75 61 61 - 62 62 - callApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 76 76 + doCallApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 63 77 const { 64 78 maxRequestRetries = 2, 65 79 retryMultiplier = DEFAULT_RETRY_MULTIPLIER ··· 109 123 } 110 124 } 111 125 126 126 + 127 127 + callApi = async <T = Response>(reqFunc: () => Request, opts: {retries?: number, logFailure?: boolean} = {}): Promise<T> => { 128 128 + const { 129 129 + maxRequestRetries = 2, 130 130 + retryMultiplier = DEFAULT_RETRY_MULTIPLIER 131 131 + } = this.config; 132 132 + 133 133 + const thisInstance = this; 134 134 + 135 135 + const { 136 136 + retries = maxRequestRetries, 137 137 + logFailure = true 138 138 + } = opts; 139 139 + 140 140 + try { 141 141 + return await pRetry(() => this.doCallApi(reqFunc()), { 142 142 + retries: retries, 143 143 + factor: retryMultiplier, 144 144 + minTimeout: 1000, 145 145 + maxRetryTime: 30000, 146 146 + onFailedAttempt(context) { 147 147 + if(logFailure) { 148 148 + thisInstance.logger.warn(new SimpleError(`Request attempt ${context.attemptNumber} failed. ${context.retriesLeft} retries left`, {cause: context.error, shortStack: true})); 149 149 + } 150 150 + }, 151 151 + shouldRetry(context) { 152 152 + const cause = findCauseByFunc<request.ResponseError>(context.error, (e) => isSuperAgentResponseError(e)); 153 153 + if(cause === undefined) { 154 154 + return false; 155 155 + } 156 156 + if([400,403,401].includes(cause.status)) { 157 157 + return false; 158 158 + } 159 159 + return true; 160 160 + } 161 161 + }) 162 162 + } catch (e) { 163 163 + throw e; 164 164 + } 165 165 + } 166 166 + 112 167 testConnection = async () => { 113 168 try { 114 169 await isPortReachableConnect(this.url.port, {host: this.url.url.hostname}); 115 115 - this.isKoito = await this.checkKoito(); 116 170 return true; 117 171 } catch (e) { 118 172 if(e.status === 410 || e.message.includes('HTTP Status 410')) { ··· 124 178 125 179 testAuth = async () => { 126 180 try { 127 127 - const resp = await this.callApi(request.get(`${joinedUrl(this.url.url,'1/validate-token')}`)); 181 181 + const resp = await this.callApi(() => request.get(`${joinedUrl(this.url.url,'1/validate-token')}`)); 128 182 return true; 129 183 } catch (e) { 130 184 throw e; ··· 133 187 134 188 async checkKoito(): Promise<boolean> { 135 189 try { 136 136 - const resp = await this.callApi(request.get(`${joinedUrl(getBaseFromUrl(this.url.url), 'apis/web/v1/stats')}`)); 137 137 - this.logger.info('Listenbrainz Host looks like a Koito server, API client will now operate in Koito mode!'); 138 138 - this.logger.warn('Koito has limited support for the Listenbrainz API spec. It does not support Now Playing or retrieving full metabrainz data for a play. Please consider switching to the full Koito Source/Client.'); 190 190 + const resp = await this.callApi(() => request.get(`${joinedUrl(getBaseFromUrl(this.url.url), 'apis/web/v1/stats')}`), {retries: 0, logFailure: false}); 139 191 return true; 140 192 } catch (e) { 141 141 - this.logger.verbose('Listenbrainz Host does not look like a Koito server.'); 142 142 - } 143 143 - return false; 144 144 - } 145 145 - 146 146 - getUserListens = async (maxTracks: number, user?: string): Promise<ListensResponse> => { 147 147 - try { 148 148 - 149 149 - const resp = await this.callApi(request 150 150 - .get(`${joinedUrl(this.url.url,'1/user',user ?? this.config.username, 'listens')}`) 151 151 - // this endpoint can take forever, sometimes, and we want to make sure we timeout in a reasonable amount of time for polling sources to continue trying to scrobble 152 152 - .timeout({ 153 153 - response: 15000, // wait 15 seconds before timeout if server doesn't response at all 154 154 - deadline: 30000 // wait 30 seconds overall for request to complete 155 155 - }) 156 156 - .query({ 157 157 - count: maxTracks 158 158 - })); 159 159 - const {body: {payload}} = resp as any; 160 160 - return payload as ListensResponse; 161 161 - } catch (e) { 162 162 - throw e; 163 163 - } 164 164 - } 165 165 - 166 166 - getUserListensKoito = async (maxTracks: number): Promise<KoitoListensResponse> => { 167 167 - try { 168 168 - 169 169 - const resp = await this.callApi(request 170 170 - .get(`${joinedUrl(getBaseFromUrl(this.url.url), '/apis/web/v1/listens')}`) 171 171 - .query({ 172 172 - period: 'all_time', 173 173 - page: 0, 174 174 - limit: maxTracks 175 175 - }) 176 176 - ); 177 177 - const { body } = resp as any; 178 178 - return body as KoitoListensResponse; 179 179 - } catch (e) { 180 180 - throw e; 193 193 + return false; 181 194 } 182 195 } 183 196 184 197 getPlayingNow = async (user?: string): Promise<ListensResponse> => { 185 198 try { 186 199 187 187 - const resp = await this.callApi(request 200 200 + const resp = await this.callApi(() => request 188 201 .get(`${joinedUrl(this.url.url,'1/user',user ?? this.config.username, 'playing-now')}`) 189 202 // this endpoint can take forever, sometimes, and we want to make sure we timeout in a reasonable amount of time for polling sources to continue trying to scrobble 190 203 .timeout({ ··· 201 214 } 202 215 } 203 216 204 204 - getRecentlyPlayed = async (maxTracks: number, user?: string): Promise<PlayObject[]> => { 205 205 - if(this.isKoito) { 206 206 - return this.getRecentlyPlayedKoito(maxTracks) 207 207 - } 208 208 - 209 209 - try { 210 210 - const resp = await this.getUserListens(maxTracks, user); 211 211 - return resp.listens.map(x => listenResponseToPlay(x)); 212 212 - } catch (e) { 213 213 - this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`); 214 214 - return []; 215 215 - } 216 216 - } 217 217 - 218 218 - getRecentlyPlayedKoito = async (maxTracks: number): Promise<PlayObject[]> => { 219 219 - try { 220 220 - const resp = await this.getUserListensKoito(maxTracks); 221 221 - return resp.items.map(x => listenObjectResponseToPlay(x)); 222 222 - } catch (e) { 223 223 - this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`); 224 224 - return []; 225 225 - } 226 226 - } 227 227 - 228 228 - 229 217 submitListen = async (play: PlayObject, options: SubmitOptions = {}): Promise<ScrobbleActionResult> => { 230 218 const listenPayload = playToSubmitPayload(play, {listenType: options.listenType}); 231 219 const { log = false} = options; ··· 237 225 // so no useful information 238 226 // https://listenbrainz.readthedocs.io/en/latest/users/api-usage.html#submitting-listens 239 227 // TODO may we should make a call to recent-listens to get the parsed scrobble? 240 240 - const resp = await this.callApi(request.post(`${joinedUrl(this.url.url,'1/submit-listens')}`).type('json').send(listenPayload)); 228 228 + const resp = await this.callApi(() => request.post(`${joinedUrl(this.url.url,'1/submit-listens')}`).type('json').send(listenPayload)); 241 229 if(log) { 242 230 this.logger.debug(`Submit Response: ${resp.text}`) 243 231 } ··· 261 249 } 262 250 return playObj; 263 251 } 252 252 + 253 253 + getUserListens = async (options: UserListensOptions & {user?: string} = {}): Promise<ListensResponse> => { 254 254 + const { user } = options; 255 255 + try { 256 256 + /** https://rain0r.github.io/listenbrainz-openapi/#/lbCore/listensForUser 257 257 + * https://listenbrainz.readthedocs.io/en/latest/users/api/core.html#get--1-user-(mb_username-user_name)-listens 258 258 + */ 259 259 + const resp = await this.callApi(() => request 260 260 + .get(`${joinedUrl(this.url.url,'1/user',user ?? this.config.username, 'listens')}`) 261 261 + // this endpoint can take forever, sometimes, and we want to make sure we timeout in a reasonable amount of time for polling sources to continue trying to scrobble 262 262 + .timeout({ 263 263 + response: 15000, // wait 15 seconds before timeout if server doesn't respond at all 264 264 + deadline: 30000 // wait 30 seconds overall for request to complete 265 265 + }) 266 266 + .query(options)); 267 267 + const {body: {payload}} = resp as any; 268 268 + return payload as ListensResponse; 269 269 + } catch (e) { 270 270 + throw new UpstreamError('Getting user listens failed', {cause: e}); 271 271 + } 272 272 + } 273 273 + 274 274 + getPagelessTimeRangeListens = async (options: PagelessListensTimeRangeOptions & {user?: string} = {}): Promise<PagelessTimeRangeListensResult> => { 275 275 + const { limit = 100, to, from, user } = options; 276 276 + 277 277 + const lzListensOptions: UserListensOptions = { 278 278 + count: Math.min(limit, MAX_ITEMS_PER_GET_LZ), 279 279 + }; 280 280 + 281 281 + if (from !== undefined) { 282 282 + lzListensOptions.min_ts = from; 283 283 + } 284 284 + if (to !== undefined) { 285 285 + lzListensOptions.max_ts = to; 286 286 + } 287 287 + 288 288 + try { 289 289 + const lr = await this.getUserListens({...lzListensOptions, user}); 290 290 + const more = to > lr.latest_listen_ts; 291 291 + return {data: lr.listens.map(x => listenResponseToPlay(x)), meta: {...options, total: lr.count, more}}; 292 292 + } catch (e) { 293 293 + throw e; 294 294 + } 295 295 + } 296 296 + 297 297 + getPaginatedUnitOfTime(): dayjs.ManipulateType { 298 298 + return 'second'; 299 299 + } 300 300 + 264 301 265 302 static formatPlayObj(obj: any, options: FormatPlayObjectOptions): PlayObject { 266 303 return listenResponseToPlay(obj);

+18 -10

src/backend/common/vendor/RockSkyApiClient.ts

Reviewed

··· 15 15 import { identifierToAtProtoHandle } from './bluesky/bsUtils.js'; 16 16 import { baseFormatPlayObj } from "../../utils/PlayTransformUtils.js"; 17 17 import { ScrobbleSubmitError } from "../errors/MSErrors.js"; 18 18 + import { tryApiCall } from "../../utils/RequestUtils.js"; 18 19 19 20 interface SubmitOptions { 20 21 log?: boolean ··· 58 59 } 59 60 60 61 61 61 - callLZApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 62 62 + doCallLZApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 62 63 try { 63 64 req.set('Authorization', `Token ${this.config.key ?? this.config.token}`); 64 65 return await req as T; ··· 103 104 } 104 105 } 105 106 106 106 - callApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 107 107 - const { 108 108 - maxRequestRetries = 2, 109 109 - retryMultiplier = DEFAULT_RETRY_MULTIPLIER 110 110 - } = this.config; 107 107 + callLZApi = async <T = Response>(reqFunc: () => Request, retries = 0): Promise<T> => { 111 108 112 109 try { 113 113 - return await req as T; 110 110 + return await tryApiCall(() => this.doCallLZApi(reqFunc()), {...this.config, logger: this.logger}) as T; 111 111 + } catch (e) { 112 112 + throw e; 113 113 + } 114 114 + } 115 115 + 116 116 + 117 117 + callApi = async <T = Response>(reqFunc: () => Request, retries = 0): Promise<T> => { 118 118 + const apiCall = async () => await reqFunc(); 119 119 + 120 120 + try { 121 121 + return await tryApiCall(apiCall, {...this.config, logger: this.logger}) as T; 114 122 } catch (e) { 115 123 throw e; 116 124 } ··· 132 140 133 141 testAuth = async () => { 134 142 try { 135 135 - const resp = await this.callLZApi(request.get(`${joinedUrl(this.lzUrl.url,'1/validate-token')}`)); 143 143 + const resp = await this.callLZApi(() => request.get(`${joinedUrl(this.lzUrl.url,'1/validate-token')}`)); 136 144 return true; 137 145 } catch (e) { 138 146 throw e; ··· 142 150 getUserListens = async (maxTracks: number, user?: string): Promise<UserScrobbleResponse> => { 143 151 try { 144 152 145 145 - const resp = await this.callApi(request 153 153 + const resp = await this.callApi(() => request 146 154 .get(`${joinedUrl(this.apiUrl.url,`app.rocksky.actor.getActorScrobbles`)}`) 147 155 // this endpoint can take forever, sometimes, and we want to make sure we timeout in a reasonable amount of time for polling sources to continue trying to scrobble 148 156 .timeout({ ··· 190 198 if(log) { 191 199 this.logger.debug(`Submit Payload: ${JSON.stringify(submitPayload)}`); 192 200 } 193 193 - const resp = await this.callLZApi(request.post(`${joinedUrl(this.lzUrl.url,'1/submit-listens')}`).type('json').send(submitPayload)); 201 201 + const resp = await this.callLZApi(() => request.post(`${joinedUrl(this.lzUrl.url,'1/submit-listens')}`).type('json').send(submitPayload)); 194 202 if(log) { 195 203 this.logger.debug(`Submit Response: ${resp.text}`) 196 204 }

+37 -10

src/backend/common/vendor/bluesky/AbstractBlueSkyApiClient.ts

Reviewed

··· 1 1 import { getRoot } from "../../../ioc.js"; 2 2 - import { AbstractApiOptions } from "../../infrastructure/Atomic.js"; 2 2 + import { AbstractApiOptions, PagelessListensTimeRangeOptions, PagelessTimeRangeListens, PagelessTimeRangeListensResult } from "../../infrastructure/Atomic.js"; 3 3 import { ListRecord, ScrobbleRecord, TealClientData } from "../../infrastructure/config/client/tealfm.js"; 4 4 import AbstractApiClient from "../AbstractApiClient.js"; 5 5 - import { Agent, ComAtprotoRepoCreateRecord } from "@atproto/api"; 5 5 + import { Agent, ComAtprotoRepoCreateRecord, ComAtprotoRepoListRecords } from "@atproto/api"; 6 6 import { MSCache } from "../../Cache.js"; 7 7 - import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult } from "../../../../core/Atomic.js"; 7 7 + import { BrainzMeta, PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, UnixTimestamp } from "../../../../core/Atomic.js"; 8 8 import { musicServiceToCononical } from "../ListenbrainzApiClient.js"; 9 9 import { parseRegexSingle } from "@foxxmd/regex-buddy-core"; 10 10 import { RecordOptions } from "../../infrastructure/config/client/tealfm.js"; 11 11 - import dayjs from "dayjs"; 12 12 - import { getScrobbleTsSOCDateWithContext } from "../../../utils/TimeUtils.js"; 11 11 + import dayjs, { ManipulateType } from "dayjs"; 12 12 + import { getScrobbleTsSOCDateWithContext, usecToUnix } from "../../../utils/TimeUtils.js"; 13 13 import { removeUndefinedKeys } from "../../../utils.js"; 14 14 import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.js"; 15 15 import { ScrobbleSubmitError } from "../../errors/MSErrors.js"; 16 16 import { UpstreamError } from "../../errors/UpstreamError.js"; 17 17 - 17 17 + import { decodeTid, generateTID } from '@ewanc26/tid'; 18 18 19 19 - export abstract class AbstractBlueSkyApiClient extends AbstractApiClient { 19 19 + export abstract class AbstractBlueSkyApiClient extends AbstractApiClient implements PagelessTimeRangeListens { 20 20 21 21 declare config: TealClientData; 22 22 ··· 48 48 } 49 49 } 50 50 51 51 - async listScrobbleRecord(limit: number = 20): Promise<ListRecord<ScrobbleRecord>[]> { 51 51 + async listScrobbleRecord(options: {limit?: number, cursor?: string} = {}): Promise<ComAtprotoRepoListRecords.Response> { 52 52 + const {limit = 20, cursor} = options; 52 53 try { 54 54 + // records are returned newest to oldest 53 55 const response = await this.agent.com.atproto.repo.listRecords({ 54 56 repo: this.agent.sessionManager.did, 55 57 collection: "fm.teal.alpha.feed.play", 56 56 - limit 58 58 + limit, 59 59 + cursor // cursor TID is EXCLUSIVE IE first record returned will be the first older than cursor 57 60 }); 58 58 - return response.data.records as unknown as ListRecord<ScrobbleRecord>[]; 61 61 + return response; 59 62 } catch (e) { 60 63 throw new UpstreamError(`Failed to list scrobble record`, { cause: e, response: 'response' in e ? e.response : undefined }); 61 64 } 65 65 + } 66 66 + 67 67 + async getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise<PagelessTimeRangeListensResult> { 68 68 + const {to, limit} = params; 69 69 + 70 70 + let cursor: string; 71 71 + if(to !== undefined) { 72 72 + cursor = generateTID(dayjs.unix(to).toISOString()); 73 73 + } 74 74 + 75 75 + const resp = await this.listScrobbleRecord({cursor, limit}); 76 76 + let fromTS: UnixTimestamp; 77 77 + if(resp.data.cursor !== undefined) { 78 78 + const { timestampUs } = decodeTid(resp.data.cursor); 79 79 + fromTS = usecToUnix(timestampUs); 80 80 + } 81 81 + 82 82 + const plays = (resp.data.records as unknown as ListRecord<ScrobbleRecord>[]).map(x => listRecordToPlay(x)); 83 83 + 84 84 + return {data: plays, meta: {to, from: fromTS, limit}}; 85 85 + } 86 86 + 87 87 + getPaginatedUnitOfTime(): ManipulateType { 88 88 + return 'second'; 62 89 } 63 90 } 64 91

+53 -21

src/backend/common/vendor/koito/KoitoApiClient.ts

Reviewed

··· 1 1 import dayjs from "dayjs"; 2 2 import { PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../../core/Atomic.js"; 3 3 - import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER } from "../../infrastructure/Atomic.js"; 4 4 - import { KoitoData, ListenObjectResponse, ListensResponse } from "../../infrastructure/config/client/koito.js"; 3 3 + import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult } from "../../infrastructure/Atomic.js"; 4 4 + import { GetListensOptions, KoitoData, ListenObjectResponse, ListensResponse } from "../../infrastructure/config/client/koito.js"; 5 5 import AbstractApiClient from "../AbstractApiClient.js"; 6 6 import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../../../utils/NetworkUtils.js"; 7 7 import request, { Request, Response } from 'superagent'; ··· 12 12 import { parseRegexSingleOrFail } from "../../../utils.js"; 13 13 import { baseFormatPlayObj } from "../../../utils/PlayTransformUtils.js"; 14 14 import { ScrobbleSubmitError } from "../../errors/MSErrors.js"; 15 15 + import { tryApiCall } from "../../../utils/RequestUtils.js"; 15 16 16 17 interface SubmitOptions { 17 18 log?: boolean ··· 20 21 21 22 const KOITO_LZ_PATH: RegExp = new RegExp(/^\/apis\/listenbrainz(\/?1?\/?)?$/); 22 23 23 23 - export class KoitoApiClient extends AbstractApiClient { 24 24 + export class KoitoApiClient extends AbstractApiClient implements PaginatedTimeRangeListens<number> { 24 25 25 26 declare config: KoitoData; 26 27 url: URLData; ··· 46 47 this.logger.verbose(`Config URL: '${url ?? '(None Given)'}' => Normalized: '${this.url.url}'`) 47 48 } 48 49 49 49 - callApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 50 50 - const { 51 51 - maxRequestRetries = 2, 52 52 - retryMultiplier = DEFAULT_RETRY_MULTIPLIER 53 53 - } = this.config; 50 50 + doCallApi = async <T = Response>(req: Request, retries = 0): Promise<T> => { 54 51 55 52 try { 56 53 req.set('Authorization', `Token ${this.config.token}`); ··· 95 92 } 96 93 } 97 94 95 95 + callApi = async <T = Response>(reqFunc: () => Request, retries = 0): Promise<T> => { 96 96 + const apiCall = async () => await reqFunc(); 97 97 + 98 98 + try { 99 99 + return await tryApiCall(() => this.doCallApi(reqFunc()), {...this.config, logger: this.logger}) as T; 100 100 + } catch (e) { 101 101 + throw e; 102 102 + } 103 103 + } 104 104 + 98 105 testConnection = async () => { 99 106 try { 100 107 await isPortReachableConnect(this.url.port, { host: this.url.url.hostname }); ··· 103 110 } 104 111 105 112 try { 106 106 - const resp = await this.callApi(request.get(`${joinedUrl(this.url.url, 'apis/web/v1/stats')}`)); 113 113 + const resp = await this.callApi(() => request.get(`${joinedUrl(this.url.url, 'apis/web/v1/stats')}`)); 107 114 if(resp.type !== 'application/json') { 108 115 throw new Error(`Expected response from ${resp.request.url} to be 'application/json' but got ${resp.type}. Is the Normalized Koito URL correct?`); 109 116 } ··· 121 128 122 129 testAuth = async () => { 123 130 try { 124 124 - const resp = await this.callApi(request.get(`${joinedUrl(this.url.url, '/apis/listenbrainz/1/validate-token')}`)); 131 131 + const resp = await this.callApi(() => request.get(`${joinedUrl(this.url.url, '/apis/listenbrainz/1/validate-token')}`)); 125 132 return true; 126 133 } catch (e) { 127 134 throw new Error('Could not validate Koito API Key', { cause: e }); 128 135 } 129 136 } 130 137 131 131 - getUserListens = async (maxTracks: number): Promise<ListensResponse> => { 138 138 + getUserListens = async (options: GetListensOptions): Promise<ListensResponse> => { 139 139 + const {page = 0} = options; 132 140 try { 133 141 134 134 - const resp = await this.callApi(request 142 142 + const resp = await this.callApi(() => request 135 143 .get(`${joinedUrl(this.url.url, '/apis/web/v1/listens')}`) 136 144 .query({ 137 145 period: 'all_time', 138 138 - page: 0, 139 139 - limit: maxTracks 146 146 + page, 147 147 + ...options 140 148 }) 141 149 ); 142 150 const { body } = resp as any; ··· 146 154 } 147 155 } 148 156 149 149 - getRecentlyPlayed = async (maxTracks: number): Promise<PlayObject[]> => { 157 157 + getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions<number>): Promise<PaginatedTimeRangeListensResult<number>> => { 158 158 + 159 159 + let resp: ListensResponse; 150 160 try { 151 151 - const resp = await this.getUserListens(maxTracks); 152 152 - return resp.items.map(x => listenObjectResponseToPlay(x, { url: this.url.url })); 153 153 - } catch (e) { 154 154 - this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`); 155 155 - return []; 161 161 + resp = await this.getUserListens({...params, page: params.cursor}); 162 162 + } catch (e) { 163 163 + throw new Error('Error occurred while getting Koito paginated listens', { cause: e }); 164 164 + } 165 165 + 166 166 + return { 167 167 + data: resp.items.map((x => listenObjectResponseToPlay(x, { url: this.url.url }))), 168 168 + meta: { 169 169 + ...params, 170 170 + total: resp.total_record_count, 171 171 + cursor: resp.current_page, 172 172 + more: resp.has_next_page 173 173 + } 156 174 } 157 175 } 158 176 177 177 + getPaginatedUnitOfTime(): dayjs.ManipulateType { 178 178 + return 'second'; 179 179 + } 180 180 + 181 181 + // getRecentlyPlayed = async (maxTracks: number): Promise<PlayObject[]> => { 182 182 + // try { 183 183 + // const resp = await this.getUserListens({limit: maxTracks}); 184 184 + // return resp.items.map(x => listenObjectResponseToPlay(x, { url: this.url.url })); 185 185 + // } catch (e) { 186 186 + // this.logger.error(`Error encountered while getting User listens | Error => ${e.message}`); 187 187 + // return []; 188 188 + // } 189 189 + // } 190 190 + 159 191 submitListen = async (play: PlayObject, options: SubmitOptions = {}): Promise<ScrobbleActionResult> => { 160 192 const { log = false, listenType = 'single' } = options; 161 193 const listenPayload: SubmitPayload = { listen_type: listenType, payload: [playToListenPayload(play)] }; ··· 170 202 // so no useful information 171 203 // https://listenbrainz.readthedocs.io/en/latest/users/api-usage.html#submitting-listens 172 204 // TODO may we should make a call to recent-listens to get the parsed scrobble? 173 173 - const resp = await this.callApi(request.post(`${joinedUrl(this.url.url, '/apis/listenbrainz/1/submit-listens')}`).type('json').send(listenPayload)); 205 205 + const resp = await this.callApi(() => request.post(`${joinedUrl(this.url.url, '/apis/listenbrainz/1/submit-listens')}`).type('json').send(listenPayload)); 174 206 if (log) { 175 207 this.logger.debug(`Submit Response: ${resp.text}`) 176 208 }

+80 -29

src/backend/common/vendor/maloja/MalojaApiClient.ts

Reviewed

··· 1 1 - import dayjs from 'dayjs'; 2 2 - import request, { SuperAgentRequest, Response } from 'superagent'; 1 1 + import dayjs, { ManipulateType } from 'dayjs'; 2 2 + import request, { Response, Request } from 'superagent'; 3 3 import compareVersions from "compare-versions"; 4 4 import AbstractApiClient from "../AbstractApiClient.js"; 5 5 import { getBaseFromUrl, isPortReachableConnect, joinedUrl, normalizeWebAddress } from "../../../utils/NetworkUtils.js"; 6 6 import { MalojaData } from "../../infrastructure/config/client/maloja.js"; 7 7 import { PlayObject, PlayObjectLifecycleless, ScrobbleActionResult, URLData } from "../../../../core/Atomic.js"; 8 8 - import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions } from "../../infrastructure/Atomic.js"; 8 8 + import { AbstractApiOptions, DEFAULT_RETRY_MULTIPLIER, FormatPlayObjectOptions, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult } from "../../infrastructure/Atomic.js"; 9 9 import { isNodeNetworkException } from "../../errors/NodeErrors.js"; 10 10 import { isSuperAgentResponseError } from "../../errors/ErrorUtils.js"; 11 11 import { getNonEmptyVal, parseRetryAfterSecsFromObj, removeUndefinedKeys, sleep } from "../../../utils.js"; ··· 15 15 import { buildTrackString } from '../../../../core/StringUtils.js'; 16 16 import { baseFormatPlayObj } from '../../../utils/PlayTransformUtils.js'; 17 17 import { ScrobbleSubmitError } from '../../errors/MSErrors.js'; 18 18 + import { NO_RETRY_HTTP_STATUS, tryApiCall } from '../../../utils/RequestUtils.js'; 18 19 19 20 20 21 21 21 - export class MalojaApiClient extends AbstractApiClient { 22 22 + export class MalojaApiClient extends AbstractApiClient implements PaginatedTimeRangeListens<number> { 22 23 23 24 declare config: MalojaData; 24 25 url: URLData; ··· 37 38 this.logger.verbose(`Config URL: '${url ?? '(None Given)'}' => Normalized: '${this.url.url}'`) 38 39 } 39 40 40 40 - callApi = async <T = Response>(req: SuperAgentRequest, retries = 0): Promise<T> => { 41 41 - const { 42 42 - maxRequestRetries = 1, 43 43 - retryMultiplier = DEFAULT_RETRY_MULTIPLIER 44 44 - } = this.config; 45 45 - 41 41 + doCallApi = async <T = Response>(req: Request): Promise<T> => { 46 42 try { 47 43 return await req as T; 48 44 } catch (e) { 49 45 if ((isNodeNetworkException(e) || isSuperAgentResponseError(e) && e.timeout)) { 50 50 - if (retries < maxRequestRetries) { 51 51 - const retryAfter = parseRetryAfterSecsFromObj(e) ?? (retryMultiplier * (retries + 1)); 52 52 - this.logger.warn(`Request failed but retries (${retries}) less than max (${maxRequestRetries}), retrying request after ${retryAfter} seconds...`); 53 53 - await sleep(retryAfter * 1000); 54 54 - return await this.callApi(req, retries + 1) 55 55 - } else { 56 56 - throw new UpstreamError(`Request continued to fail after reach max retries (${maxRequestRetries})`, { cause: e, showStopper: true }); 57 57 - } 46 46 + throw new UpstreamError(`API Call failed`, { cause: e }); 58 47 } else if (isSuperAgentResponseError(e)) { 59 48 const { 60 49 message, ··· 75 64 } 76 65 } 77 66 67 67 + callApi = async <T = Response>(reqFunc: () => Request): Promise<T> => { 68 68 + try { 69 69 + return await tryApiCall(() => this.doCallApi(reqFunc()), { 70 70 + ...this.config, 71 71 + logger: this.logger, 72 72 + noRetryStatus: [...NO_RETRY_HTTP_STATUS, 409] 73 73 + }) as T; 74 74 + } catch (e) { 75 75 + throw e; 76 76 + } 77 77 + } 78 78 + 78 79 testConnection = async () => { 79 80 try { 80 81 await isPortReachableConnect(this.url.port, { host: this.url.url.hostname }); ··· 83 84 } 84 85 85 86 try { 86 86 - const serverInfoResp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/serverinfo`)); 87 87 + const serverInfoResp = await this.callApi(() => request.get(`${this.url.url}/apis/mlj_1/serverinfo`)); 87 88 const { 88 89 statusCode, 89 90 body: { ··· 119 120 testHealth = async () => { 120 121 121 122 try { 122 122 - const serverInfoResp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/serverinfo`), 0); 123 123 + const serverInfoResp = await this.callApi(() => request.get(`${this.url.url}/apis/mlj_1/serverinfo`)); 123 124 const { 124 125 statusCode, 125 126 body: { ··· 151 152 152 153 testAuth = async () => { 153 154 try { 154 154 - const resp = await this.callApi(request 155 155 + const resp = await this.callApi(() => request 155 156 .get(`${this.url.url}/apis/mlj_1/test`) 156 157 .query({ key: this.config.apiKey })); 157 158 ··· 179 180 } 180 181 } 181 182 182 182 - getRecentScrobbles = async (limit: number) => { 183 183 - const resp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/scrobbles?perpage=${limit}`)); 184 184 - const { 185 185 - body: { 186 186 - list = [], 187 187 - } = {}, 183 183 + // getRecentScrobbles = async (limit: number) => { 184 184 + // const resp = await this.callApi(request.get(`${this.url.url}/apis/mlj_1/scrobbles?perpage=${limit}`)); 185 185 + // const { 186 186 + // body: { 187 187 + // list = [], 188 188 + // } = {}, 189 189 + // } = resp; 190 190 + // return list.map(formatPlayObj); 191 191 + // } 192 192 + 193 193 + getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions<number>): Promise<PaginatedTimeRangeListensResult<number>> => { 194 194 + 195 195 + const opts: RecentlyPlayedRequestOptions = { 196 196 + perpage: params.limit, 197 197 + page: params.cursor, 198 198 + from: params.from !== undefined ? dayjs.unix(params.from).format('YYYY/MM/DD') : undefined, 199 199 + until: params.to !== undefined ? dayjs.unix(params.from).format('YYYY/MM/DD') : undefined 200 200 + }; 201 201 + 202 202 + const resp = await this.getScrobbles(opts); 203 203 + 204 204 + return { 205 205 + data: resp.list.map(x => formatPlayObj(x)), 206 206 + meta: {...params, more: resp.pagination.next_page !== null} 207 207 + } 208 208 + } 209 209 + 210 210 + getScrobbles = async (options: RecentlyPlayedRequestOptions = {}): Promise<RecentlyPlayedResponse> => { 211 211 + const resp = await this.callApi(() => request.get(`${this.url.url}/apis/mlj_1/scrobbles`).query(removeUndefinedKeys(options))); 212 212 + const { 213 213 + body 188 214 } = resp; 189 189 - return list.map(formatPlayObj); 215 215 + 216 216 + return body as RecentlyPlayedResponse; 217 217 + } 218 218 + 219 219 + getPaginatedUnitOfTime(): ManipulateType { 220 220 + return 'day'; 190 221 } 191 222 192 223 scrobble = async (playObj: PlayObject): Promise<ScrobbleActionResult> => { ··· 211 242 try { 212 243 213 244 214 214 - const response = await this.callApi(request.post(`${this.url.url}/apis/mlj_1/newscrobble`) 245 245 + const response = await this.callApi(() => request.post(`${this.url.url}/apis/mlj_1/newscrobble`) 215 246 .type('json') 216 247 .send(scrobbleData)); 217 248 ··· 438 469 // https://github.com/FoxxMD/multi-scrobbler/issues/454#issuecomment-3806367420 439 470 440 471 return scrobbleData; 472 472 + } 473 473 + 474 474 + export interface RecentlyPlayedRequestOptions { 475 475 + page?: number; 476 476 + perpage?: number; 477 477 + /** start of timerange, smallest granularity like '2026/02/10 */ 478 478 + from?: string; 479 479 + /** end of timerange, smallest granularity like '2026/02/10 */ 480 480 + until?: string; 481 481 + } 482 482 + 483 483 + export interface RecentlyPlayedResponse { 484 484 + list: MalojaScrobbleData[]; 485 485 + pagination: { 486 486 + next_page: null |string 487 487 + prev_page: null | string 488 488 + page: number 489 489 + perpage: number 490 490 + } 491 491 + status: string; 441 492 }

+23 -23

src/backend/common/vendor/musicbrainz/CoverArtApiClient.ts

Reviewed

··· 8 8 import { joinedUrl } from "../../../utils/NetworkUtils.js"; 9 9 import { hasNodeNetworkException } from "../../errors/NodeErrors.js"; 10 10 import { sleep } from "../../../utils.js"; 11 11 + import { RequestRetryOptions } from "../../infrastructure/config/common.js"; 12 12 + import { RetryContext } from "p-retry"; 13 13 + import { NO_RETRY_HTTP_STATUS, tryApiCall } from "../../../utils/RequestUtils.js"; 11 14 12 15 export type ThumbSize = 250 | 500 | 1200; 13 16 const THUMB_SIZES = [250, 500, 1200]; ··· 41 44 images: CoverArtReleaseImage[] 42 45 } 43 46 44 44 - export interface CoverArtApiConfig { 47 47 + export interface CoverArtApiConfig extends RequestRetryOptions { 45 48 url?: URL; 46 49 } 47 50 ··· 83 86 return cachedArt; 84 87 } else { 85 88 let result = undefined, 86 86 - retries = 0, 87 89 err: Error; 88 90 89 91 const url = joinedUrl(this.baseUrl, `/release/${mbid}/${thumbParams}`); 90 92 91 91 - while(result === undefined && retries < 2) { 92 92 - try { 93 93 - const resp = await this.coverThumbRequest(url.toString()); 94 94 - if(resp === undefined) { 95 95 - result = false; 96 96 - } else { 97 97 - result = resp; 98 98 - } 99 99 - err = undefined; 100 100 - } catch (e) { 101 101 - err = e; 102 102 - if(hasNodeNetworkException(e)) { 103 103 - this.logger.warn(`Request to ${url.toString()} failed but retries (${retries}) are not greater than max (1), retrying request after a short break... Error Message: ${e.message}`); 104 104 - retries++; 105 105 - await sleep(500); 106 106 - continue; 107 107 - } else { 108 108 - break; 109 109 - } 93 93 + try { 94 94 + const resp = await this.coverThumbRequest(url.toString()); 95 95 + if(resp === undefined) { 96 96 + result = false; 97 97 + } else { 98 98 + result = resp; 110 99 } 100 100 + err = undefined; 101 101 + } catch (e) { 102 102 + err = e; 111 103 } 112 104 113 105 if(result === false) { ··· 127 119 protected coverThumbRequest = async (url: string): Promise<string | undefined> => { 128 120 try { 129 121 // https://musicbrainz.org/doc/Cover_Art_Archive/API#/release/{mbid}/({id}|front|back)-(250|500|1200) 130 130 - const resp = await request 122 122 + const resp = await tryApiCall(() => request 131 123 .get(url) 132 124 // only follow first redirect so we get the url without actually downloading the image 133 133 - .redirects(1); 125 125 + .redirects(1), { 126 126 + ...this.config, 127 127 + logFailure: logUnexpectedStatus, 128 128 + noRetryStatus: [...NO_RETRY_HTTP_STATUS, 404, 302, 307] 129 129 + }); 134 130 throw new Error('Should not be getting this far'); 135 131 } catch (e) { 136 132 if (isSuperAgentResponseError(e)) { ··· 173 169 } 174 170 } 175 171 } 172 172 + } 173 173 + 174 174 + const logUnexpectedStatus = (context: RetryContext): boolean => { 175 175 + return !isSuperAgentResponseError(context.error) || ![404,302,307].includes(context.error.status); 176 176 }

+2

src/backend/index.ts

Reviewed

··· 6 6 import relativeTime from 'dayjs/plugin/relativeTime.js'; 7 7 import isToday from 'dayjs/plugin/isToday.js'; 8 8 import timezone from 'dayjs/plugin/timezone.js'; 9 9 + import week from 'dayjs/plugin/weekOfYear.js'; 9 10 import utc from 'dayjs/plugin/utc.js'; 10 11 import * as path from "path"; 11 12 import { SimpleIntervalJob, ToadScheduler } from "toad-scheduler"; ··· 29 30 dayjs.extend(duration); 30 31 dayjs.extend(timezone); 31 32 dayjs.extend(isToday); 33 33 + dayjs.extend(week); 32 34 33 35 // eslint-disable-next-line prefer-arrow-functions/prefer-arrow-functions 34 36 (async function () {

+212 -208

src/backend/scrobblers/AbstractScrobbleClient.ts

Reviewed

··· 9 9 NowPlayingUpdateThreshold, 10 10 PlayObject, 11 11 PlayObjectLifecycleless, 12 12 - QueuedScrobble, ScrobbleActionResult, PlayMatchResult, ScrobblePayload, ScrobbleResponse, SourcePlayerObj, TA_DURING, 12 12 + QueuedScrobble, ScrobbleActionResult, PlayMatchResult, SourcePlayerObj, TA_DURING, 13 13 TA_FUZZY, 14 14 - TrackStringOptions 14 14 + TrackStringOptions, 15 15 + TA_EXACT, 16 16 + SOURCE_SOT 15 17 } from "../../core/Atomic.js"; 16 18 import { buildTrackString, capitalize, truncateStringToLength } from "../../core/StringUtils.js"; 17 19 import AbstractComponent from "../common/AbstractComponent.js"; 18 18 - import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.js"; 20 20 + import { hasUpstreamError } from "../common/errors/UpstreamError.js"; 19 21 import { 20 22 ARTIST_WEIGHT, 21 23 Authenticatable, ··· 24 26 DEFAULT_RETRY_MULTIPLIER, 25 27 DUP_SCORE_THRESHOLD, 26 28 FormatPlayObjectOptions, 27 27 - PlayPlatformId, 29 29 + PaginatedTimeRangeOptions, 30 30 + REFRESH_STALE_DEFAULT, 28 31 ScrobbledPlayObject, 29 32 SourceIdentifier, 30 33 TIME_WEIGHT, 34 34 + TimeRangeListensFetcher, 31 35 TITLE_WEIGHT, 32 36 } from "../common/infrastructure/Atomic.js"; 33 37 import { CommonClientConfig, NowPlayingOptions, UpstreamRefreshOptions } from "../common/infrastructure/config/client/index.js"; ··· 35 39 import { Notifiers } from "../notifier/Notifiers.js"; 36 40 import { 37 41 comparingMultipleArtists, 38 38 - genGroupId, 39 39 - genGroupIdStr, 40 40 - genGroupIdStrFromPlay, 41 42 isDebugMode, 42 43 parseBool, 43 44 playObjDataMatch, 44 45 pollingBackoff, 45 45 - setIntersection, 46 46 sleep, 47 47 sortByOldestPlayDate, 48 48 } from "../utils.js"; ··· 52 52 hasAcceptableTemporalAccuracy, 53 53 temporalAccuracyToString, 54 54 temporalPlayComparisonSummary, 55 55 + todayAwareFormat, 55 56 } from "../utils/TimeUtils.js"; 56 57 import { WebhookPayload } from "../common/infrastructure/config/health/webhooks.js"; 57 58 import { AsyncTask, SimpleIntervalJob, Task, ToadScheduler } from "toad-scheduler"; 58 58 - import { MSCache } from "../common/Cache.js"; 59 59 import { getRoot } from "../ioc.js"; 60 60 import { rehydratePlay } from "../utils/CacheUtils.js"; 61 61 import { findAsyncSequential, staggerMapper } from "../utils/AsyncUtils.js"; ··· 63 63 import { comparePlayArtistsNormalized, comparePlayTracksNormalized, lifecyclelessInvariantTransform } from "../utils/PlayComparisonUtils.js"; 64 64 import { normalizeStr } from "../utils/StringUtils.js"; 65 65 import prom, { Counter, Gauge } from 'prom-client'; 66 66 - import { ScrobbleSubmitError } from "../common/errors/MSErrors.js"; 66 66 + import { ScrobbleSubmitError, SimpleError } from "../common/errors/MSErrors.js"; 67 67 import {serializeError} from 'serialize-error'; 68 68 - import { redactString } from "@foxxmd/redact-string"; 69 69 - import clone from "clone"; 68 68 + import { DEFAULT_NEW_PADDING, groupPlaysToTimeRanges } from "../utils/ListenFetchUtils.js"; 70 69 71 70 type PlatformMappedPlays = Map<string, {player: SourcePlayerObj, source: SourceIdentifier}>; 72 71 type NowPlayingQueue = Map<string, PlatformMappedPlays>; ··· 83 82 protected MAX_STORED_SCROBBLES = 40; 84 83 protected MAX_INITIAL_SCROBBLES_FETCH = this.MAX_STORED_SCROBBLES; 85 84 86 86 - #recentScrobblesList: PlayObject[] = []; 85 85 + scrobbleSOTRanges: PaginatedTimeRangeOptions[] = []; 87 86 scrobbledPlayObjs: FixedSizeList<ScrobbledPlayObject>; 88 88 - lastScrobbledPlayDate?: Dayjs; 89 89 - newestScrobbleTime?: Dayjs 90 90 - oldestScrobbleTime?: Dayjs 91 87 tracksScrobbled: number = 0; 92 88 93 93 - lastScrobbleCheck: Dayjs = dayjs(0) 94 89 lastScrobbleAttempt: Dayjs = dayjs(0) 95 90 upstreamRefresh: MarkOptional<Required<UpstreamRefreshOptions>, 'refreshInitialCount'>; 96 91 checkExistingScrobbles: boolean; ··· 142 137 refreshEnabled = true, 143 138 refreshInitialCount, 144 139 refreshMinInterval = 5, 145 145 - refreshStaleAfter = 60, 140 140 + refreshStaleAfter = REFRESH_STALE_DEFAULT, 146 141 checkExistingScrobbles = true, 147 142 verbose = {}, 148 143 } = {}, ··· 183 178 this.queuedGauge = clientMetrics.queued; 184 179 this.deadLetterGauge = clientMetrics.deadLetter; 185 180 this.scrobbledCounter = clientMetrics.scrobbled; 186 186 - } 187 187 - 188 188 - set recentScrobbles(scrobbles: PlayObject[]) { 189 189 - const sorted = [...scrobbles]; 190 190 - sorted.sort(sortByOldestPlayDate); 191 191 - this.#recentScrobblesList = sorted; 192 192 - } 193 193 - 194 194 - get recentScrobbles() { 195 195 - return this.#recentScrobblesList; 196 181 } 197 182 198 183 protected getIdentifier() { ··· 359 344 this.initializeNowPlaying(); 360 345 361 346 let initialLimit = refreshInitialCount; 362 362 - if(refreshInitialCount > this.MAX_INITIAL_SCROBBLES_FETCH) { 347 347 + if (refreshInitialCount > this.MAX_INITIAL_SCROBBLES_FETCH) { 363 348 this.logger.warn(`Defined initial scrobbles count (${refreshInitialCount}) higher than maximum allowed (${this.MAX_INITIAL_SCROBBLES_FETCH}). Will use max instead.`); 364 349 initialLimit = this.MAX_INITIAL_SCROBBLES_FETCH; 365 350 } 366 351 367 367 - this.logger.verbose(`Fetching up to ${initialLimit} initial scrobbles...`); 368 368 - await this.refreshScrobbles(initialLimit); 369 369 - this.lastScrobbledPlayDate = this.newestScrobbleTime; 370 370 - } 352 352 + this.logger.verbose(`Preloading up to ${initialLimit} initial scrobbles...`); 371 353 372 372 - refreshScrobbles = async (limit: number = this.MAX_STORED_SCROBBLES) => { 373 373 - if (this.upstreamRefresh.refreshEnabled) { 374 374 - this.logger.debug('Refreshing recent scrobbles'); 375 375 - const recent = await this.getScrobblesForRefresh(limit); 376 376 - this.logger.debug(`Found ${recent.length} recent scrobbles`); 377 377 - this.recentScrobbles = recent; 378 378 - if (this.recentScrobbles.length > 0) { 379 379 - const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1); 380 380 - const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1); 381 381 - this.newestScrobbleTime = newestScrobbleTime; 382 382 - this.oldestScrobbleTime = oldestScrobbleTime; 383 383 - 384 384 - this.filterScrobbledTracks(); 354 354 + try { 355 355 + const preload = await this.getScrobblesForTimeRange({ 356 356 + limit: initialLimit, 357 357 + fetchMax: initialLimit 358 358 + }); 359 359 + if(preload === undefined) { 360 360 + this.logger.warn('Preload result was undefined!'); 361 361 + } else { 362 362 + if(preload.length === 0) { 363 363 + this.logger.verbose(`Preloaded 0 scrobbles.`); 364 364 + } else { 365 365 + preload.sort(sortByOldestPlayDate); 366 366 + const from = preload[0].data.playDate; 367 367 + // we are assuming that all fetchers return latest scrobbles first (pretty sure this is the case) 368 368 + const to = dayjs();// preload[preload.length - 1].data.playDate; 369 369 + await this.cache.cacheClientScrobbles.set<PlayObject[]>(this.getScrobbleCacheKey(from, to), preload, '60s'); 370 370 + this.scrobbleSOTRanges.push({from: from.unix(), to: to.unix()}); 371 371 + this.logger.verbose(`Preloaded ${preload.length} scrobbles from ${todayAwareFormat(from)} to ${todayAwareFormat(to)}`); 372 372 + } 385 373 } 374 374 + } catch (e) { 375 375 + this.logger.warn(new SimpleError('Could not preload scrobbles', {cause: e, shortStack: true})); 386 376 } 387 387 - this.lastScrobbleCheck = dayjs(); 388 377 } 389 378 390 390 - protected abstract getScrobblesForRefresh(limit: number): Promise<PlayObject[]>; 379 379 + abstract getScrobblesForTimeRange: TimeRangeListensFetcher; 391 380 392 392 - shouldRefreshScrobble = () => { 393 393 - const { 394 394 - refreshStaleAfter, 395 395 - refreshMinInterval, 396 396 - refreshEnabled 397 397 - } = this.upstreamRefresh; 381 381 + protected getScrobbleCacheKey = (from: Dayjs | number, to: Dayjs | number): string => { 382 382 + return `${this.name}-scrobbleRange-${typeof from === 'number' ? from : from.unix()}-${typeof to === 'number' ? to :to.unix()}`; 383 383 + } 398 384 399 399 - if (!refreshEnabled) { 400 400 - this.logger.debug({labels: ['Upstream Refresh']}, `Should NOT refresh => refreshEnabled is false`); 401 401 - return false; 402 402 - } 385 385 + handleQueuedScrobbleRanges = () => { 386 386 + this.scrobbleSOTRanges = groupPlaysToTimeRanges(this.queuedScrobbles.map(x => x.play).concat(this.deadLetterScrobbles.map(x => x.play)), this.scrobbleSOTRanges, {staleNowBuffer: this.config.options?.refreshStaleAfter}); 387 387 + } 403 388 404 404 - if(this.queuedScrobbles.length === 0) { 405 405 - this.logger.debug({labels: ['Upstream Refresh']}, `Should NOT refresh => no scrobbles in queue!`); 406 406 - return false; 389 389 + getSOTScrobblesForPlay = async (play: PlayObject): Promise<PlayObject[]> => { 390 390 + let range: PaginatedTimeRangeOptions = this.scrobbleSOTRanges.find(x => x.from <= play.data.playDate.unix() && x.to > Math.min(dayjs().subtract(this.config.options?.refreshStaleAfter ?? REFRESH_STALE_DEFAULT, 's').unix(), play.data.playDate.unix())); 391 391 + if(range === undefined) { 392 392 + this.logger.warn(`No Scrobble SOT range found! Should have been handled before this. Creating a new one for ${buildTrackString(play)}`); 393 393 + range = { 394 394 + from: play.data.playDate.subtract(DEFAULT_NEW_PADDING).unix(), 395 395 + to: Math.min(play.data.playDate.add(DEFAULT_NEW_PADDING).unix(), dayjs().subtract(this.config.options?.refreshStaleAfter ?? REFRESH_STALE_DEFAULT, 's').unix()) 396 396 + }; 397 397 + this.scrobbleSOTRanges.push(range); 407 398 } 408 408 - 409 409 - const queuedPlayedDate = this.getLatestQueuePlayDate(); 410 410 - 411 411 - // if newest queued play was played more recently than the last time we refreshed upstream scrobbles 412 412 - if (this.scrobblesLastCheckedAt().unix() < queuedPlayedDate.unix()) { 413 413 - if(!this.scrobblesRefreshMinIntervalPassed()) { 414 414 - this.logger.debug({labels: ['Upstream Refresh']}, `Should refresh but WILL NOT => queued scrobble playDate is newer than last refresh but refreshMinInterval (${refreshMinInterval}ms) has not passed since last check`); 415 415 - return false; 416 416 - } else { 417 417 - this.logger.debug({labels: ['Upstream Refresh']}, 'Should refresh => newest queued scrobble playDate is newer than last refresh'); 418 418 - return true; 419 419 - } 399 399 + const cachedPlaysRes = await this.cache.cacheClientScrobbles.get<PlayObject[] | Error>(this.getScrobbleCacheKey(range.from, range.to)); 400 400 + if(cachedPlaysRes instanceof Error) { 401 401 + throw new SimpleError('Cannot get historical plays due to cached error', {cause: cachedPlaysRes, shortStack: true}); 420 402 } 421 421 - 422 422 - // if the play date of the last Play scrobbled is *newer* 423 423 - // than the queued scrobble we are about to scrobble 424 424 - // then we are inserting a scrobble out of order which can happen if 425 425 - // * backlogging and upstream returned plays out of order 426 426 - // * processing dead letter queue 427 427 - // * two sources have different history 428 428 - // 429 429 - // in all cases we probably want to refresh 430 430 - if(this.lastScrobbledPlayDate !== undefined && this.queuedScrobbles[0].play.meta.newFromSource && this.lastScrobbledPlayDate.unix() > this.queuedScrobbles[0].play.data.playDate.unix()) { 431 431 - if(!this.scrobblesRefreshMinIntervalPassed()) { 432 432 - this.logger.debug({labels: ['Upstream Refresh']}, `Should refresh but WILL NOT => queued scrobble playDate is older than last scrobbled play (out-of-order insert) but refreshMinInterval (${refreshMinInterval}ms) has not passed since last check`); 433 433 - return false; 434 434 - } else { 435 435 - this.logger.debug({labels: ['Upstream Refresh']}, 'Should refresh => queued scrobble playDate is older than last scrobbled play (out-of-order insert)'); 436 436 - return true; 437 437 - } 403 403 + if(cachedPlaysRes !== undefined) { 404 404 + return cachedPlaysRes; 438 405 } 439 439 - 440 440 - // if it's been X seconds since we last refreshed 441 441 - if(refreshStaleAfter !== undefined) { 442 442 - const diff = dayjs().diff(this.scrobblesLastCheckedAt(), 's'); 443 443 - if(diff > refreshStaleAfter) { 444 444 - this.logger.debug({labels: ['Upstream Refresh']}, `Should refresh => last refresh (${diff}s ago) was longer than refreshStaleAfter (${refreshStaleAfter}s)`); 445 445 - return true; 446 446 - } 406 406 + try { 407 407 + const plays = await this.getScrobblesForTimeRange(range); 408 408 + plays.sort(sortByOldestPlayDate); 409 409 + await this.cache.cacheClientScrobbles.set<PlayObject[] | Error>(this.getScrobbleCacheKey(range.from, range.to), plays, (this.config.options?.refreshStaleAfter ?? REFRESH_STALE_DEFAULT) * 1000); 410 410 + return plays; 411 411 + } catch (e) { 412 412 + await this.cache.cacheClientScrobbles.set<PlayObject[] | Error>(this.getScrobbleCacheKey(range.from, range.to), e, '10s'); 413 413 + throw new SimpleError('Cannot get historical plays', {cause: e, shortStack: true}); 447 414 } 448 448 - 449 449 - return false; 450 415 } 451 451 - 452 452 - protected scrobblesLastCheckedAt = () => this.lastScrobbleCheck 453 453 - protected scrobblesLastCheckedAtDiff = () => dayjs().diff(this.scrobblesLastCheckedAt(), 'ms') 454 454 - protected scrobblesRefreshMinIntervalPassed = () => { 455 455 - const { 456 456 - refreshMinInterval, 457 457 - } = this.upstreamRefresh; 458 458 - return this.scrobblesLastCheckedAtDiff() >= refreshMinInterval; 459 459 - } 460 460 - 461 416 public async alreadyScrobbled(playObj: PlayObject, log?: boolean): Promise<[boolean, PlayMatchResult]> { 462 462 - const result = await this.existingScrobble(playObj); 417 417 + const result = await this.existingScrobble(playObj, await this.getSOTScrobblesForPlay(playObj)); 463 418 return [result.match, result]; 464 419 } 465 420 ··· 468 423 return obj; 469 424 } 470 425 471 471 - // time frame is valid as long as the play date for the source track is newer than the oldest play time from the scrobble client 472 472 - // ...this is assuming the scrobble client is returning "most recent" scrobbles 473 473 - timeFrameIsValid = (playObj: PlayObject) => { 474 474 - 475 475 - if(this.oldestScrobbleTime === undefined) { 476 476 - return [true, '']; 477 477 - } 478 478 - 479 479 - const { 480 480 - data: { 481 481 - playDate, 482 482 - } = {}, 483 483 - } = playObj; 484 484 - const validTime = playDate.isAfter(this.oldestScrobbleTime); 485 485 - let log = ''; 486 486 - if (!validTime) { 487 487 - const dur = dayjs.duration(Math.abs(playDate.diff(this.oldestScrobbleTime))).humanize(false); 488 488 - log = `occurred ${dur} before the oldest scrobble returned by this client (${this.oldestScrobbleTime.format()})`; 489 489 - } 490 490 - return [validTime, log] 491 491 - } 492 492 - 493 426 addScrobbledTrack = (playObj: PlayObject, scrobbledPlay: PlayObjectLifecycleless) => { 494 427 this.scrobbledPlayObjs.add({play: playObj, scrobble: scrobbledPlay}); 495 428 this.scrobbledCounter.labels(this.getPrometheusLabels()).inc(); 496 496 - this.lastScrobbledPlayDate = playObj.data.playDate; 429 429 + //this.lastScrobbledPlayDate = playObj.data.playDate; 497 430 this.tracksScrobbled++; 498 498 - } 499 499 - 500 500 - filterScrobbledTracks = () => { 501 501 - this.scrobbledPlayObjs = new FixedSizeList<ScrobbledPlayObject>(this.MAX_STORED_SCROBBLES, this.scrobbledPlayObjs.data.filter(x => this.timeFrameIsValid(x.play)[0])) ; 502 431 } 503 432 504 433 getScrobbledPlays = () => this.scrobbledPlayObjs.data.map(x => x.play) ··· 523 452 return [matchPlayDate, dtInvariantMatches]; 524 453 } 525 454 526 526 - existingScrobble = async (playObjPre: PlayObject): Promise<PlayMatchResult> => { 455 455 + existingScrobble = async (playObjPre: PlayObject, existingScrobbles: PlayObject[], log: boolean = true): Promise<PlayMatchResult> => { 527 456 528 457 const result: PlayMatchResult = { 529 458 match: false, ··· 568 497 569 498 // if no recent scrobbles found then assume we haven't submitted it 570 499 // (either user doesnt want to check history or there is no history to check!) 571 571 - if (this.recentScrobbles.length === 0) { 572 572 - this.dupeLogger.trace(`${buildTrackString(playObj, scoreTrackOpts)} => No Match because no recent scrobbles returned from API`); 500 500 + if (existingScrobbles.length === 0) { 501 501 + this.dupeLogger.trace(`${buildTrackString(playObj, scoreTrackOpts)} => No Match because no existing scrobbles returned from API`); 573 502 result.reason = 'no recent scrobbles returned from API'; 574 503 return result; 575 504 } 576 505 577 577 - // we have found an existing submission but without an exact date 578 578 - // in which case we can check the scrobble api response against recent scrobbles (also from api) for a more accurate comparison 579 579 - //const referenceApiScrobbleResponse = existingDataSubmitted.length > 0 ? existingDataSubmitted[0].scrobble : undefined; 580 580 - 581 506 // only check for fuzzy if we know this play is NOT a repeat 582 507 // otherwise we may get a false positive on the previously played track ending time == repeat start time 583 508 // -- this is info we only know if play was generated from MS player so we can be reasonably sure 584 584 - const looseTimeAccuracy = playObj.data.repeat ? [TA_DURING] : [TA_FUZZY, TA_DURING]; 509 509 + // 510 510 + // OR if play was generated from a source that uses History (endpoint sources, lfm or lz history sources) 511 511 + // then we can be reasonably sure that our candidate play has an accurate timestamp and wouldn't fuzzy match a previous scrobble 512 512 + const looseTimeAccuracy = playObj.data.repeat || playObj.meta.sourceSOT === SOURCE_SOT.HISTORY ? [TA_DURING] : [TA_FUZZY, TA_DURING]; 585 513 586 514 587 587 - existingScrobble = await findAsyncSequential(this.recentScrobbles, async (xPre) => { 515 515 + existingScrobble = await findAsyncSequential(existingScrobbles, async (xPre) => { 588 516 589 517 const x = await this.transformPlay(xPre, TRANSFORM_HOOK.existing); 590 518 ··· 651 579 result.match = score >= DUP_SCORE_THRESHOLD; 652 580 result.breakdowns = scoreBreakdowns; 653 581 result.score = score; 654 654 - //closestMatch = scoreInfo 582 582 + 583 583 + if(result.match === false && temporalComparison.match === TA_EXACT && score >= 0.90) { 584 584 + // if we have a score >= 90 and time is an exact match 585 585 + // it's likely the differences are due to source-scrobbler data presentation, or deficiencies, 586 586 + // rather than actually being unique 587 587 + // so force match in this instance 588 588 + result.match = true; 589 589 + result.reason = `Score ${score.toFixed(2)} is not greater than threshold (${DUP_SCORE_THRESHOLD}) but it is very close and timestamp is an exact match, vibe matching.`; 590 590 + } 655 591 } 656 592 657 593 return score >= DUP_SCORE_THRESHOLD; ··· 663 599 closestScrobbleParts.push(`Closest Scrobble: ${buildTrackString(result.closestMatchedPlay, scoreTrackOpts)}`); 664 600 } 665 601 closestScrobbleParts.push(result.reason); 666 666 - let summary = `${capitalize(playObj.meta.source ?? 'Source')}: ${buildTrackString(playObj, scoreTrackOpts)} => ${closestScrobbleParts.join(' => ')}`; 667 667 - this.dupeLogger.trace(`${summary}${result.breakdowns.length > 0 ? `\n${result.breakdowns.join('\n')}` : ''}`); 668 668 - 602 602 + let summaryStart = `${capitalize(playObj.meta.source ?? 'Source')}: ${buildTrackString(playObj, scoreTrackOpts)} => ${closestScrobbleParts.join(' => ')}`; 603 603 + const summary = `${summaryStart}${result.breakdowns.length > 0 ? `\n${result.breakdowns.join('\n')}` : ''}` 604 604 + result.summary = summary; 605 605 + if(log) { 606 606 + this.dupeLogger.trace(summary); 607 607 + } 669 608 return result; 670 609 } 671 610 ··· 809 748 810 749 try { 811 750 this.scrobbling = true; 751 751 + if(!this.upstreamRefresh.refreshEnabled) { 752 752 + this.logger.verbose('Scrobble refresh is DISABLED. All queued scrobbles will likely always be scrobbled (nothing to check duplicates against).'); 753 753 + } 812 754 while (!this.shouldStopScrobbleProcessing()) { 755 755 + let queueEmpty = this.queuedScrobbles.length === 0; 813 756 while (this.queuedScrobbles.length > 0) { 814 814 - if (this.shouldRefreshScrobble()) { 815 815 - await this.refreshScrobbles(); 757 757 + this.handleQueuedScrobbleRanges(); 758 758 + if(!this.upstreamRefresh.refreshEnabled) { 759 759 + this.logger.trace('Scrobble refresh is DISABLED.'); 816 760 } 761 761 + 817 762 const currQueuedPlay = this.queuedScrobbles.shift(); 818 763 819 819 - const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(currQueuedPlay.play); 820 820 - if (timeFrameValid) { 821 821 - const [matched, matchResult] = await this.alreadyScrobbled(currQueuedPlay.play); 764 764 + let historicalPlays: PlayObject[] = []; 765 765 + let historicalError: Error | undefined; 766 766 + 767 767 + if(this.upstreamRefresh.refreshEnabled) { 768 768 + try { 769 769 + historicalPlays = await this.getSOTScrobblesForPlay(currQueuedPlay.play); 770 770 + } catch (e) { 771 771 + historicalError = e; 772 772 + if(e.message === 'Cannot get historical plays due to cached error') { 773 773 + this.logger.warn(`${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' => Previous error while getting historical scrobbles means this scrobble cannot be compared, will queue as dead for now.`); 774 774 + this.logger.trace(e); 775 775 + } else { 776 776 + this.logger.warn(new SimpleError(`${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' => cannot get historical scrobbles, will queue as dead for now.`, {cause: e, shortStack: true})); 777 777 + } 778 778 + this.addDeadLetterScrobble(currQueuedPlay, e); 779 779 + } 780 780 + } 781 781 + if(historicalError === undefined) { 782 782 + const {summary, ...matchResult} = await this.existingScrobble(currQueuedPlay.play, historicalPlays); 822 783 const { 823 784 scrobble = {}, 824 785 ...lifeRest ··· 830 791 match: matchResult 831 792 } 832 793 } 833 833 - if(!matched) { 794 794 + if(!matchResult.match) { 834 795 const transformedScrobble = await this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.postCompare); 835 796 if(transformedScrobble.meta.lifecycle === undefined) { 836 797 transformedScrobble.meta.lifecycle = { ··· 866 827 } 867 828 } 868 829 } 869 869 - } else if (!timeFrameValid) { 870 870 - this.logger.debug(`Will not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' because it ${timeFrameValidLog}`); 871 830 } 872 831 this.updateQueuedScrobblesCache(); 873 832 this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedScrobbles.length); 874 833 this.emitEvent('scrobbleDequeued', {queuedScrobble: currQueuedPlay}) 834 834 + } 835 835 + if(!queueEmpty) { 836 836 + this.emitEvent('queueEmptied', {}); 875 837 } 876 838 await sleep(this.scrobbleSleep); 877 839 } ··· 905 867 const processable = this.deadLetterScrobbles.filter(x => x.retries < retries); 906 868 const queueStatus = `${processable.length} of ${this.deadLetterScrobbles.length} dead scrobbles have less than ${retries} retries, ${processable.length === 0 ? 'will skip processing.': 'processing now...'}`; 907 869 if (processable.length === 0) { 908 908 - this.logger.verbose(queueStatus, {leaf: 'Dead Letter'}); 870 870 + this.logger.verbose({labels: 'Dead Letter'}, queueStatus); 909 871 return; 910 872 } 911 911 - this.logger.info(queueStatus, {leaf: 'Dead Letter'}); 873 873 + this.logger.info({labels: 'Dead Letter'}, queueStatus); 874 874 + if(!this.upstreamRefresh.refreshEnabled) { 875 875 + this.logger.verbose({labels: 'Dead Letter'}, 'Scrobble refresh is DISABLED. All dead scrobbles will likely always be scrobbled (nothing to check duplicates against).'); 876 876 + } 877 877 + this.handleQueuedScrobbleRanges(); 912 878 913 879 const removedIds = []; 914 880 for (const deadScrobble of this.deadLetterScrobbles) { ··· 920 886 } 921 887 } 922 888 if (removedIds.length > 0) { 923 923 - this.logger.info(`Removed ${removedIds.length} scrobbles from dead letter queue`, {leaf: 'Dead Letter'}); 889 889 + this.logger.info({labels: 'Dead Letter'}, `Removed ${removedIds.length} scrobbles from dead letter queue`); 924 890 } 925 891 } 926 892 ··· 929 895 const deadScrobble = this.deadLetterScrobbles[deadScrobbleIndex]; 930 896 931 897 if (!(await this.isReady())) { 932 932 - this.logger.warn('Cannot process dead letter scrobble because client is not ready.', {leaf: 'Dead Letter'}); 898 898 + this.logger.warn({labels: 'Dead Letter'}, 'Cannot process dead letter scrobble because client is not ready.'); 933 899 return [false, deadScrobble]; 934 900 } 935 935 - if (this.getLatestQueuePlayDate() !== undefined && this.scrobblesLastCheckedAt().unix() < this.getLatestQueuePlayDate().unix()) { 936 936 - await this.refreshScrobbles(); 937 937 - } 938 938 - const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(deadScrobble.play); 939 939 - if (timeFrameValid) { 940 940 - const [matched, matchResult] = await this.alreadyScrobbled(deadScrobble.play); 941 941 - const { 942 942 - scrobble = {}, 943 943 - ...lifeRest 944 944 - } = deadScrobble.play.meta.lifecycle ?? {steps: [], original: deadScrobble.play}; 945 945 - deadScrobble.play.meta.lifecycle = { 946 946 - ...lifeRest, 947 947 - scrobble: { 948 948 - ...scrobble, 949 949 - match: matchResult[1] 901 901 + let historicalPlays: PlayObject[] = []; 902 902 + if(this.upstreamRefresh.refreshEnabled) { 903 903 + try { 904 904 + historicalPlays = await this.getSOTScrobblesForPlay(deadScrobble.play); 905 905 + } catch (e) { 906 906 + if(e.message === 'Cannot get historical plays due to cached error') { 907 907 + this.logger.warn(`${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' => Previous error while getting historical scrobbles means this scrobble cannot be compared`); 908 908 + this.logger.trace(e); 909 909 + } else { 910 910 + this.logger.warn(new SimpleError(`${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' => cannot get historical scrobbles`, {cause: e, shortStack: true})); 950 911 } 912 912 + deadScrobble.retries++; 913 913 + deadScrobble.error = messageWithCauses(e); 914 914 + deadScrobble.lastRetry = dayjs(); 915 915 + this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; 916 916 + this.updateDeadLetterCache(); 917 917 + return [false, deadScrobble]; 918 918 + } finally { 919 919 + await sleep(1000); 951 920 } 952 952 - if(!matched) { 953 953 - const transformedScrobble = await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare); 954 954 - try { 955 955 - const scrobbledPlay = await this.scrobble(transformedScrobble); 956 956 - this.emitEvent('scrobble', {play: transformedScrobble}); 957 957 - this.addScrobbledTrack(transformedScrobble, scrobbledPlay); 958 958 - } catch (e) { 921 921 + } 922 922 + const {summary, ...matchResult} = await this.existingScrobble(deadScrobble.play, historicalPlays); 923 923 + const { 924 924 + scrobble = {}, 925 925 + ...lifeRest 926 926 + } = deadScrobble.play.meta.lifecycle ?? {steps: [], original: deadScrobble.play}; 927 927 + deadScrobble.play.meta.lifecycle = { 928 928 + ...lifeRest, 929 929 + scrobble: { 930 930 + ...scrobble, 931 931 + match: matchResult 932 932 + } 933 933 + } 934 934 + if(!matchResult.match) { 935 935 + const transformedScrobble = await this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare); 936 936 + try { 937 937 + const scrobbledPlay = await this.scrobble(transformedScrobble); 938 938 + this.emitEvent('scrobble', {play: transformedScrobble}); 939 939 + this.addScrobbledTrack(transformedScrobble, scrobbledPlay); 940 940 + } catch (e) { 959 941 960 960 - const submitError = findCauseByReference(e, ScrobbleSubmitError); 961 961 - if(submitError !== undefined) { 962 962 - deadScrobble.play.meta.lifecycle.scrobble.payload = submitError.payload; 963 963 - deadScrobble.play.meta.lifecycle.scrobble.response = submitError.responseBody; 964 964 - deadScrobble.play.meta.lifecycle.scrobble.error = serializeError(submitError); 965 965 - } else { 966 966 - deadScrobble.play.meta.lifecycle.scrobble.payload = this.playToClientPayload(transformedScrobble); 967 967 - deadScrobble.play.meta.lifecycle.scrobble.error = serializeError(e); 968 968 - } 942 942 + const submitError = findCauseByReference(e, ScrobbleSubmitError); 943 943 + if(submitError !== undefined) { 944 944 + deadScrobble.play.meta.lifecycle.scrobble.payload = submitError.payload; 945 945 + deadScrobble.play.meta.lifecycle.scrobble.response = submitError.responseBody; 946 946 + deadScrobble.play.meta.lifecycle.scrobble.error = serializeError(submitError); 947 947 + } else { 948 948 + deadScrobble.play.meta.lifecycle.scrobble.payload = this.playToClientPayload(transformedScrobble); 949 949 + deadScrobble.play.meta.lifecycle.scrobble.error = serializeError(e); 950 950 + } 969 951 970 970 - deadScrobble.retries++; 971 971 - deadScrobble.error = messageWithCauses(e); 972 972 - deadScrobble.lastRetry = dayjs(); 973 973 - this.logger.error(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.source}' due to error`, {cause: e})); 974 974 - this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; 975 975 - return [false, deadScrobble]; 976 976 - } finally { 977 977 - await sleep(1000); 978 978 - } 952 952 + deadScrobble.retries++; 953 953 + deadScrobble.error = messageWithCauses(e); 954 954 + deadScrobble.lastRetry = dayjs(); 955 955 + this.logger.error(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.source}' due to error`, {cause: e})); 956 956 + this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; 957 957 + this.updateDeadLetterCache(); 958 958 + return [false, deadScrobble]; 959 959 + } finally { 960 960 + await sleep(1000); 979 961 } 980 980 - } else if (!timeFrameValid) { 981 981 - this.logger.debug(`Will not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' because it ${timeFrameValidLog}`, {leaf: 'Dead Letter'}); 982 962 } 983 963 if(deadScrobble !== undefined) { 984 964 this.removeDeadLetterScrobble(deadScrobble.id) ··· 1016 996 const plays = Array.isArray(data) ? data : [data]; 1017 997 const sm = staggerMapper<PlayObject, PlayObject>({concurrency: 2}); 1018 998 for await(const play of pMapIterable(plays, sm(async x => await this.transformPlay(x, TRANSFORM_HOOK.preCompare)), {concurrency: 2})) { 999 999 + try { 1000 1000 + const existingQueued = await this.existingScrobble(play, this.queuedScrobbles.map(x => x.play), false); 1001 1001 + // want to be very confident of this 1002 1002 + if(existingQueued.match && existingQueued.score > 0.99) { 1003 1003 + this.logger.trace(`Not adding to queue because it is already in the queue\n${existingQueued.summary}`); 1004 1004 + return; 1005 1005 + } 1006 1006 + } catch (e) { 1007 1007 + this.logger.warn(new SimpleError('Failed to check queued scrobble for existing before adding', {cause: e})); 1008 1008 + } 1019 1009 const queuedPlay = {id: nanoid(), source, play: play} 1020 1010 this.emitEvent('scrobbleQueued', {queuedPlay: queuedPlay}); 1021 1011 this.queuedScrobbles.push(queuedPlay); ··· 1024 1014 this.queuedScrobbles.sort((a, b) => sortByOldestPlayDate(a.play, b.play)); 1025 1015 } 1026 1016 this.updateQueuedScrobblesCache(); 1017 1017 + } 1018 1018 + 1019 1019 + cancelQueuedItemsBySource = (source: string): number => { 1020 1020 + const beforeMain = this.queuedScrobbles.length; 1021 1021 + const beforeDead = this.deadLetterScrobbles.length; 1022 1022 + 1023 1023 + this.queuedScrobbles = this.queuedScrobbles.filter(item => item.source !== source); 1024 1024 + this.deadLetterScrobbles = this.deadLetterScrobbles.filter(item => item.source !== source); 1025 1025 + 1026 1026 + this.updateQueuedScrobblesCache(); 1027 1027 + this.updateDeadLetterCache(); 1028 1028 + 1029 1029 + return (beforeMain + beforeDead) - (this.queuedScrobbles.length + this.deadLetterScrobbles.length); 1027 1030 } 1028 1031 1029 1032 protected addDeadLetterScrobble = (data: QueuedScrobble<PlayObject>, error: (Error | string) = 'Unspecified error') => { ··· 1141 1144 1142 1145 protected updateDeadLetterCache = () => { 1143 1146 this.cache.cacheScrobble.set(`${this.getMachineId()}-dead`, this.deadLetterScrobbles) 1144 1144 - .then(() => isDebugMode() ? this.logger.debug('Updated dead letter cache') : null) 1147 1147 + .then(() => null) 1145 1148 .catch((e) => this.logger.warn(new Error('Error while updating dead letter cache', {cause: e}))); 1146 1149 } 1147 1150 1148 1151 protected updateQueuedScrobblesCache = () => { 1149 1152 this.cache.cacheScrobble.set(`${this.getMachineId()}-queue`, this.queuedScrobbles) 1150 1150 - .then(() => isDebugMode() ? this.logger.debug('Updated queued scrobble cache') : null) 1153 1153 + .then(() => null) 1151 1154 .catch((e) => this.logger.warn(new Error('Error while updating queued scrobble cache', {cause: e}))); 1152 1155 } 1153 1156 } ··· 1160 1163 } 1161 1164 1162 1165 export const shouldUpdatePlayingNowPlatformWhenPlayingOnly = async (data: SourcePlayerObj): Promise<boolean> => { 1163 1163 - return data.status.calculated === CALCULATED_PLAYER_STATUSES.playing; 1166 1166 + return (data.status.calculated === CALCULATED_PLAYER_STATUSES.playing) 1167 1167 + || (data.nowPlayingMode && !CALCULATED_PLAYER_STATUSES.stopped); 1164 1168 }

+14 -7

src/backend/scrobblers/DiscordScrobbler.ts

Reviewed

··· 1 1 import { Logger } from "@foxxmd/logging"; 2 2 import EventEmitter from "events"; 3 3 import { PlayMatchResult, PlayObject, SourcePlayerObj } from "../../core/Atomic.js"; 4 4 - import { CALCULATED_PLAYER_STATUSES, FormatPlayObjectOptions, REPORTED_PLAYER_STATUSES, ReportedPlayerStatus, SINGLE_USER_PLATFORM_ID_STR } from "../common/infrastructure/Atomic.js"; 4 4 + import { CALCULATED_PLAYER_STATUSES, FormatPlayObjectOptions, REPORTED_PLAYER_STATUSES, ReportedPlayerStatus, SINGLE_USER_PLATFORM_ID_STR, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 5 5 import { Notifiers } from "../notifier/Notifiers.js"; 6 6 7 7 import AbstractScrobbleClient, { nowPlayingUpdateByPlayDuration } from "./AbstractScrobbleClient.js"; ··· 29 29 this.nowPlayingMaxThreshold = nowPlayingUpdateByPlayDuration; 30 30 this.nowPlayingMinThreshold = (_) => 5; 31 31 } 32 32 + 33 33 + getScrobblesForTimeRange = async (_) => []; 32 34 33 35 formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => obj; 34 36 ··· 114 116 } 115 117 } 116 118 117 117 - getScrobblesForRefresh = async (limit: number) => { 118 118 - return []; 119 119 - } 120 120 - 121 119 queueScrobble = async (data: PlayObject | PlayObject[], source: string) => { 122 120 // discord does not handle scrobbles, only Now Playing 123 121 // so don't bother queueing any scrobbles as we don't want to cache them ··· 160 158 161 159 shouldUpdatePlayingNowPlatformSpecific = async (data: SourcePlayerObj) => { 162 160 if ([CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused, CALCULATED_PLAYER_STATUSES.playing].includes(data.status.calculated as ReportedPlayerStatus) 161 161 + || (data.nowPlayingMode && !CALCULATED_PLAYER_STATUSES.stopped) 163 162 || data.status.stale) { 164 163 165 164 const [sendOk, reasons, level = 'warn'] = await this.api.checkOkToSend(); 166 165 if (!sendOk) { 167 167 - this.logger[level](`Cannot update playing now because api client is ${reasons}`); 166 166 + this.npLogger[level](`Cannot update playing now because api client is ${reasons}`); 168 167 return false; 169 168 } 170 169 171 170 if(this.api instanceof DiscordWSClient) { 172 171 const [allowed, reason] = this.api.presenceIsAllowed(); 173 172 if(!allowed) { 174 174 - this.logger.debug(reason); 173 173 + this.npLogger.debug(reason); 175 174 } 176 175 177 176 return true; 178 177 } 179 178 180 179 return true; 180 180 + } else { 181 181 + if(!data.nowPlayingMode && ![CALCULATED_PLAYER_STATUSES.stopped, CALCULATED_PLAYER_STATUSES.paused, CALCULATED_PLAYER_STATUSES.playing].includes(data.status.calculated as ReportedPlayerStatus)) { 182 182 + this.npLogger.trace(`Will not update because player is not in state: stopped | paused | playing => Found '${data.status.calculated }'`); 183 183 + } else if(data.nowPlayingMode && CALCULATED_PLAYER_STATUSES.stopped) { 184 184 + this.npLogger.trace(`Will not update because now playing player is stopped => Found ${data.status.calculated}`); 185 185 + } else { 186 186 + this.npLogger.trace('Will not update because now player is in an unexpected state for discord usage'); 187 187 + } 181 188 } 182 189 } 183 190 }

+6 -7

src/backend/scrobblers/KoitoScrobbler.ts

Reviewed

··· 4 4 import { buildTrackString, capitalize } from "../../core/StringUtils.js"; 5 5 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 6 6 import { UpstreamError } from "../common/errors/UpstreamError.js"; 7 7 - import { FormatPlayObjectOptions } from "../common/infrastructure/Atomic.js"; 7 7 + import { FormatPlayObjectOptions, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 8 8 import { playToListenPayload } from "../common/vendor/ListenbrainzApiClient.js"; 9 9 import { Notifiers } from "../notifier/Notifiers.js"; 10 10 11 11 - import AbstractScrobbleClient, { shouldUpdatePlayingNowPlatformWhenPlayingOnly } from "./AbstractScrobbleClient.js"; 11 11 + import AbstractScrobbleClient from "./AbstractScrobbleClient.js"; 12 12 import { isDebugMode } from "../utils.js"; 13 13 import { KoitoClientConfig } from "../common/infrastructure/config/client/koito.js"; 14 14 import { KoitoApiClient, listenObjectResponseToPlay } from "../common/vendor/koito/KoitoApiClient.js"; 15 15 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 16 16 + import dayjs from "dayjs"; 15 17 16 18 export default class KoitoScrobbler extends AbstractScrobbleClient { 17 19 18 20 api: KoitoApiClient; 19 21 requiresAuth = true; 20 22 requiresAuthInteraction = false; 21 21 - 23 23 + getScrobblesForTimeRange: TimeRangeListensFetcher 22 24 declare config: KoitoClientConfig; 23 25 24 26 constructor(name: any, config: KoitoClientConfig, options = {}, notifier: Notifiers, emitter: EventEmitter, logger: Logger) { ··· 28 30 // 1000 is way too high. maxing at 100 29 31 this.MAX_INITIAL_SCROBBLES_FETCH = 100; 30 32 this.supportsNowPlaying = true; 33 33 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 31 34 } 32 35 33 36 formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => listenObjectResponseToPlay(obj, options); ··· 64 67 } 65 68 throw e; 66 69 } 67 67 - } 68 68 - 69 69 - getScrobblesForRefresh = async (limit: number) => { 70 70 - return await this.api.getRecentlyPlayed(limit); 71 70 } 72 71 73 72 doScrobble = async (playObj: PlayObject) => {

+6 -7

src/backend/scrobblers/LastfmScrobbler.ts

Reviewed

··· 1 1 import { Logger } from "@foxxmd/logging"; 2 2 + import dayjs, { Dayjs } from "dayjs"; 2 3 import EventEmitter from "events"; 3 4 import { PlayObject, SourcePlayerObj } from "../../core/Atomic.js"; 4 5 import { buildTrackString, capitalize } from "../../core/StringUtils.js"; 5 6 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 6 6 - import { UpstreamError } from "../common/errors/UpstreamError.js"; 7 7 - import { FormatPlayObjectOptions, InternalConfigOptional } from "../common/infrastructure/Atomic.js"; 7 7 + import { FormatPlayObjectOptions, InternalConfigOptional, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 8 8 import { LastfmClientConfig } from "../common/infrastructure/config/client/lastfm.js"; 9 9 - import LastfmApiClient, { LastFMIgnoredScrobble, playToClientPayload, formatPlayObj, LASTFM_HOST, LASTFM_PATH } from "../common/vendor/LastfmApiClient.js"; 9 9 + import LastfmApiClient, { LastFMIgnoredScrobble, playToClientPayload, formatPlayObj } from "../common/vendor/LastfmApiClient.js"; 10 10 import { Notifiers } from "../notifier/Notifiers.js"; 11 11 import AbstractScrobbleClient, { nowPlayingUpdateByPlayDuration, shouldUpdatePlayingNowPlatformWhenPlayingOnly } from "./AbstractScrobbleClient.js"; 12 12 import { findCauseByReference } from "../utils/ErrorUtils.js"; 13 13 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 13 14 14 15 export default class LastfmScrobbler extends AbstractScrobbleClient { 15 16 ··· 17 18 requiresAuth = true; 18 19 requiresAuthInteraction = true; 19 20 upstreamType: string = 'Last.fm'; 21 21 + getScrobblesForTimeRange: TimeRangeListensFetcher 20 22 21 23 declare config: LastfmClientConfig; 22 24 ··· 28 30 this.supportsNowPlaying = true; 29 31 // last.fm shows Now Playing for the same time as the duration of the track being submitted 30 32 this.nowPlayingMaxThreshold = nowPlayingUpdateByPlayDuration; 33 33 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 31 34 } 32 35 33 36 formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => formatPlayObj(obj, options); ··· 55 58 } 56 59 throw e; 57 60 } 58 58 - } 59 59 - 60 60 - getScrobblesForRefresh = async (limit: number) => { 61 61 - return await this.api.getRecentTracks({limit}); 62 61 } 63 62 64 63 cleanSourceSearchTitle = (playObj: PlayObject) => {

+13 -12

src/backend/scrobblers/ListenbrainzScrobbler.ts

Reviewed

··· 1 1 import { Logger } from "@foxxmd/logging"; 2 2 + import dayjs, { Dayjs } from "dayjs"; 2 3 import EventEmitter from "events"; 3 4 import { PlayObject, SourcePlayerObj } from "../../core/Atomic.js"; 4 5 import { buildTrackString, capitalize } from "../../core/StringUtils.js"; 5 6 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 6 7 import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.js"; 7 7 - import { FormatPlayObjectOptions } from "../common/infrastructure/Atomic.js"; 8 8 - import { ListenBrainzClientConfig } from "../common/infrastructure/config/client/listenbrainz.js"; 8 8 + import { FormatPlayObjectOptions, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 9 9 + import { DEFAULT_MS_ITEMS_PER_GET_LZ, ListenBrainzClientConfig } from "../common/infrastructure/config/client/listenbrainz.js"; 9 10 import { ListenbrainzApiClient, playToListenPayload, playToSubmitPayload } from "../common/vendor/ListenbrainzApiClient.js"; 10 11 import { ListenPayload } from '../common/vendor/listenbrainz/interfaces.js'; 11 12 import { Notifiers } from "../notifier/Notifiers.js"; 12 13 13 14 import AbstractScrobbleClient, { nowPlayingUpdateByPlayDuration, shouldUpdatePlayingNowPlatformWhenPlayingOnly } from "./AbstractScrobbleClient.js"; 14 15 import { isDebugMode } from "../utils.js"; 16 16 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 15 17 16 18 export default class ListenbrainzScrobbler extends AbstractScrobbleClient { 17 19 18 18 - api: ListenbrainzApiClient; 20 20 + api: ListenbrainzApiClient;d 19 21 requiresAuth = true; 20 22 requiresAuthInteraction = false; 21 21 - isKoito: boolean = false; 22 22 - 23 23 + getScrobblesForTimeRange: TimeRangeListensFetcher 23 24 declare config: ListenBrainzClientConfig; 24 25 25 26 constructor(name: any, config: ListenBrainzClientConfig, options = {}, notifier: Notifiers, emitter: EventEmitter, logger: Logger) { ··· 27 28 this.api = new ListenbrainzApiClient(name, config.data, {logger: this.logger}); 28 29 // https://listenbrainz.readthedocs.io/en/latest/users/api/core.html#get--1-user-(user_name)-listens 29 30 // 1000 is way too high. maxing at 100 30 30 - this.MAX_INITIAL_SCROBBLES_FETCH = 100; 31 31 + this.MAX_INITIAL_SCROBBLES_FETCH = DEFAULT_MS_ITEMS_PER_GET_LZ; 31 32 this.supportsNowPlaying = true; 32 33 // listenbrainz shows Now Playing for the same time as the duration of the track being submitted 33 34 this.nowPlayingMaxThreshold = nowPlayingUpdateByPlayDuration; 35 35 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 34 36 } 35 37 36 38 formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => ListenbrainzApiClient.formatPlayObj(obj, options); ··· 49 51 50 52 protected async doCheckConnection(): Promise<true | string | undefined> { 51 53 await this.api.testConnection(); 52 52 - this.isKoito = this.api.isKoito; 54 54 + const isKoito = await this.api.checkKoito(); 55 55 + if(isKoito) { 56 56 + throw new Error('The given URL looks like a Koito instance. Use the Koito Scrobbler instead of Listenbrainz.') 57 57 + } 53 58 return true; 54 59 } 55 60 ··· 64 69 throw e; 65 70 } 66 71 } 67 67 - 68 68 - getScrobblesForRefresh = async (limit: number) => { 69 69 - return await this.api.getRecentlyPlayed(limit); 70 70 - } 71 71 - 72 72 + 72 73 public playToClientPayload(playObj: PlayObject): ListenPayload { 73 74 return playToListenPayload(playObj); 74 75 }

+6 -6

src/backend/scrobblers/MalojaScrobbler.ts

Reviewed

··· 4 4 import { PlayObject } from "../../core/Atomic.js"; 5 5 import { buildTrackString, capitalize } from "../../core/StringUtils.js"; 6 6 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 7 7 - import { FormatPlayObjectOptions } from "../common/infrastructure/Atomic.js"; 7 7 + import { FormatPlayObjectOptions, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 8 8 import { MalojaClientConfig } from "../common/infrastructure/config/client/maloja.js"; 9 9 import { 10 10 MalojaScrobbleRequestData, ··· 13 13 import AbstractScrobbleClient from "./AbstractScrobbleClient.js"; 14 14 import { MalojaApiClient, formatPlayObj as formatMalojaScrobbleToPlay, playToScrobblePayload } from "../common/vendor/maloja/MalojaApiClient.js"; 15 15 import { ScrobbleSubmitError } from "../common/errors/MSErrors.js"; 16 16 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 17 17 + import dayjs from "dayjs"; 16 18 17 19 const feat = ["ft.", "ft", "feat.", "feat", "featuring", "Ft.", "Ft", "Feat.", "Feat", "Featuring"]; 18 20 ··· 23 25 webUrl: string; 24 26 25 27 api: MalojaApiClient; 28 28 + getScrobblesForTimeRange: TimeRangeListensFetcher 26 29 27 30 declare config: MalojaClientConfig 28 31 ··· 30 33 super('maloja', name, config, notifier, emitter, logger); 31 34 this.api = new MalojaApiClient(name, this.config.data, { logger: childLogger(this.logger, 'API') }); 32 35 this.MAX_INITIAL_SCROBBLES_FETCH = 100; 36 36 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 33 37 } 34 38 35 39 formatPlayObj = (obj: any, options: FormatPlayObjectOptions = {}) => formatMalojaScrobbleToPlay(obj, { url: this.webUrl }); ··· 75 79 throw e; 76 80 } 77 81 } 78 78 - 79 79 - getScrobblesForRefresh = async (limit: number) => { 80 80 - return await this.api.getRecentScrobbles(limit); 81 81 - } 82 82 - 82 82 + 83 83 public playToClientPayload(playObj: PlayObject): MalojaScrobbleRequestData { 84 84 85 85 const { apiKey } = this.config.data;

+2 -2

src/backend/scrobblers/RockskyScrobbler.ts

Reviewed

··· 66 66 } 67 67 } 68 68 69 69 - getScrobblesForRefresh = async (limit: number) => { 70 70 - return await this.api.getRecentlyPlayed(limit); 69 69 + getScrobblesForTimeRange = async (_) => { 70 70 + return await this.api.getRecentlyPlayed(this.MAX_INITIAL_SCROBBLES_FETCH); 71 71 } 72 72 73 73 public playToClientPayload(playObj: PlayObject): ListenPayload {

+4 -6

src/backend/scrobblers/TealfmScrobbler.ts

Reviewed

··· 3 3 import { PlayObject } from "../../core/Atomic.js"; 4 4 import { buildTrackString, capitalize } from "../../core/StringUtils.js"; 5 5 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 6 6 - import { hasUpstreamError, UpstreamError } from "../common/errors/UpstreamError.js"; 7 6 import { FormatPlayObjectOptions } from "../common/infrastructure/Atomic.js"; 8 7 import { playToListenPayload } from "../common/vendor/ListenbrainzApiClient.js"; 9 8 import { Notifiers } from "../notifier/Notifiers.js"; 10 9 11 10 import AbstractScrobbleClient from "./AbstractScrobbleClient.js"; 12 12 - import { ListRecord, ScrobbleRecord, TealClientConfig } from "../common/infrastructure/config/client/tealfm.js"; 11 11 + import { TealClientConfig } from "../common/infrastructure/config/client/tealfm.js"; 13 12 import { BlueSkyAppApiClient } from "../common/vendor/bluesky/BlueSkyAppApiClient.js"; 14 13 import { BlueSkyOauthApiClient } from "../common/vendor/bluesky/BlueSkyOauthApiClient.js"; 15 14 import { AbstractBlueSkyApiClient, listRecordToPlay, playToRecord, recordToPlay } from "../common/vendor/bluesky/AbstractBlueSkyApiClient.js"; ··· 92 91 } 93 92 } 94 93 95 95 - getScrobblesForRefresh = async (limit: number) => { 96 96 - let list: ListRecord<ScrobbleRecord>[]; 94 94 + getScrobblesForTimeRange = async (_) => { 97 95 try { 98 98 - list = await this.client.listScrobbleRecord(limit) 96 96 + const {data} = await this.client.getPagelessTimeRangeListens({limit: 100}) 97 97 + return data; 99 98 } catch (e) { 100 99 throw new Error('Error occurred while trying to fetch records', {cause: e}); 101 100 } 102 102 - return list.map(x => listRecordToPlay(x)); 103 101 } 104 102 105 103 doScrobble = async (playObj: PlayObject) => {

+1 -5

src/backend/server/api.ts

Reviewed

··· 66 66 //let logWebLevel: LogLevel = logger.level as LogLevel || (process.env.LOG_LEVEL || 'info') as LogLevel; 67 67 68 68 const logConfig: LogOutputConfig = { 69 69 - level: logger.level as LogLevel || (process.env.LOG_LEVEL || 'info') as LogLevel, 69 69 + level: logger.level as LogLevel || (process.env.LOG_LEVEL || 'trace') as LogLevel, 70 70 sort: 'descending', 71 71 limit: 50, 72 72 - } 73 73 - if(logConfig.level === 'trace') { 74 74 - // init web logging with debug so its easier to see startup logs 75 75 - logConfig.level = 'debug'; 76 72 } 77 73 78 74 let logObjectStream: Transform;

+17 -16

src/backend/sources/AbstractSource.ts

Reviewed

··· 1 1 - import { childLogger, LogDataPretty } from '@foxxmd/logging'; 1 1 + import { childLogger, LogDataPretty, LogLevel } from '@foxxmd/logging'; 2 2 import dayjs, { Dayjs } from "dayjs"; 3 3 import { EventEmitter } from "events"; 4 4 import { FixedSizeList } from "fixed-size-list"; ··· 440 440 let checkCount = 0; 441 441 let checksOverThreshold = 0; 442 442 443 443 - const {checkActiveFor = 300, maxInterval = DEFAULT_POLLING_MAX_INTERVAL} = this.config.data; 443 443 + const {checkActiveFor = 120, maxInterval = DEFAULT_POLLING_MAX_INTERVAL} = this.config.data; 444 444 + let isInactive = false; 444 445 445 446 try { 446 447 this.polling = true; 447 448 while (!this.shouldStopPolling()) { 448 449 const pollFrom = dayjs(); 450 450 + let lastActivityLogLevel: LogLevel = 'trace'; 449 451 450 452 let playObjs: PlayObject[]; 451 453 try { ··· 481 483 }); 482 484 } 483 485 484 484 - const debugMsgs: string[] = []; 486 486 + const activityMsgs: string[] = []; 485 487 486 488 if(newDiscovered.length > 0) { 487 489 // only update date if the play date is after the current activity date (in the case of backlogged plays) ··· 494 496 const inactiveFor = dayjs.duration(Math.abs(activeThreshold.diff(dayjs(), 'millisecond'))).humanize(false); 495 497 const relativeActivity = dayjs.duration(this.lastActivityAt.diff(dayjs(), 'ms')); 496 498 const humanRelativeActivity = relativeActivity.asSeconds() > -3 ? '' : ` (${timeToHumanTimestamp(relativeActivity)} ago)`; 497 497 - let friendlyInterval = ''; 499 499 + let friendlyInterval = `${formatNumber(sleepTime)}`; 498 500 const friendlyLastFormat = todayAwareFormat(this.lastActivityAt); 501 501 + activityMsgs.push(`Last activity at ${friendlyLastFormat}${humanRelativeActivity}`); 499 502 if (activeThreshold.isBefore(dayjs())) { 500 503 friendlyInterval = formatNumber(maxInterval); 501 504 checksOverThreshold++; ··· 505 508 friendlyInterval = `(${interval} + ${backoff})`; 506 509 sleepTime = interval + backoff; 507 510 } 508 508 - if(isDebugMode()) { 509 509 - debugMsgs.push(`Last activity ${friendlyLastFormat}${humanRelativeActivity} is ${inactiveFor} outside of polling period (last activity + ${checkActiveFor}s)`); 510 510 - } else { 511 511 - debugMsgs.push(`Last activity was at ${friendlyLastFormat}${humanRelativeActivity}`); 511 511 + if(!isInactive) { 512 512 + lastActivityLogLevel = 'debug'; 513 513 + isInactive = true; 512 514 } 513 513 - } else { 514 514 - debugMsgs.push(`Last activity was at ${friendlyLastFormat}${humanRelativeActivity}`); 515 515 - friendlyInterval = `${formatNumber(sleepTime)}s`; 515 515 + activityMsgs.push(`Inactive for ${inactiveFor} (last + ${checkActiveFor}s)`); 516 516 + } else if(isInactive) { 517 517 + activityMsgs.push('New Activity after inactive period'); 518 518 + lastActivityLogLevel = 'debug'; 519 519 + isInactive = false; 516 520 } 517 517 - debugMsgs.push(`Next check in ${friendlyInterval}`); 518 518 - if(newDiscovered.length === 0) { 519 519 - debugMsgs.push('No new tracks discovered'); 520 520 - } 521 521 - this.logger.debug(debugMsgs.join(' | ')); 521 521 + activityMsgs.push(`Next check in ${friendlyInterval}s`); 522 522 + this.logger[lastActivityLogLevel](activityMsgs.join(' | ')); 522 523 this.setWakeAt(pollFrom.add(sleepTime, 'seconds')); 523 524 this.setIsSleeping(true); 524 525 while(!this.shouldStopPolling() && dayjs().isBefore(this.getWakeAt())) {

+1

src/backend/sources/EndpointLastfmSource.ts

Reviewed

··· 87 87 export const playStateFromRequest = (obj: LastFMScrobbleRequestPayload): PlayerStateData => { 88 88 89 89 const play = scrobblePayloadToPlay(obj); 90 90 + play.meta.sourceSOT = SOURCE_SOT.HISTORY; 90 91 return { 91 92 platformId: [play.meta.deviceId, NO_USER], 92 93 play,

+1

src/backend/sources/EndpointListenbrainzSource.ts

Reviewed

··· 111 111 } = obj; 112 112 113 113 const play = listenPayloadToPlay(payload[0], listen_type === 'playing_now'); 114 114 + play.meta.sourceSOT = SOURCE_SOT.HISTORY; 114 115 return { 115 116 platformId: [play.meta.deviceId, NO_USER], 116 117 play,

+8 -7

src/backend/sources/KoitoSource.ts

Reviewed

··· 1 1 import EventEmitter from "events"; 2 2 - import request from "superagent"; 3 2 import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; 4 3 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 5 5 - import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; 6 6 - import { ListenBrainzSourceConfig } from "../common/infrastructure/config/source/listenbrainz.js"; 7 7 - import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient.js"; 4 4 + import { FormatPlayObjectOptions, InternalConfig, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 8 5 import { RecentlyPlayedOptions } from "./AbstractSource.js"; 9 6 import MemorySource from "./MemorySource.js"; 10 10 - import { isPortReachableConnect } from "../utils/NetworkUtils.js"; 11 7 import { KoitoApiClient, listenObjectResponseToPlay } from "../common/vendor/koito/KoitoApiClient.js"; 12 8 import { KoitoSourceConfig } from "../common/infrastructure/config/source/koito.js"; 9 9 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 13 10 14 11 export default class KoitoSource extends MemorySource { 15 12 16 13 api: KoitoApiClient; 17 14 requiresAuth = true; 18 15 requiresAuthInteraction = false; 16 16 + getScrobblesForTimeRange: TimeRangeListensFetcher 19 17 20 18 declare config: KoitoSourceConfig; 21 19 ··· 35 33 this.supportsUpstreamRecentlyPlayed = true 36 34 this.SCROBBLE_BACKLOG_COUNT = 100; 37 35 this.logger.info(`Note: The player for this source is an analogue for the 'Now Playing' status exposed by ${this.type} which is NOT used for scrobbling. Instead, the 'recently played' or 'history' information provided by this source is used for scrobbles.`) 36 36 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 38 37 } 39 38 40 39 static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}){ return listenObjectResponseToPlay(obj, options); } ··· 62 61 getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { 63 62 const {limit = 20} = options; 64 63 await this.processRecentPlays([]); 65 65 - return await this.api.getRecentlyPlayed(limit); 64 64 + const resp = await this.getScrobblesForTimeRange({limit, cursor: 0 }); 65 65 + return resp; 66 66 } 67 67 68 68 getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => { 69 69 try { 70 70 - return await this.api.getRecentlyPlayed(20); 70 70 + const resp = await this.getScrobblesForTimeRange({limit: 20, cursor: 0 }); 71 71 + return resp; 71 72 } catch (e) { 72 73 throw e; 73 74 }

+10 -9

src/backend/sources/LastfmSource.ts

Reviewed

··· 1 1 - import dayjs from "dayjs"; 2 1 import EventEmitter from "events"; 3 3 - import request from "superagent"; 4 2 import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; 5 5 - import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 6 6 - import { FormatPlayObjectOptions, InternalConfig, PlayPlatformId, SourceType } from "../common/infrastructure/Atomic.js"; 3 3 + import { FormatPlayObjectOptions, InternalConfig, PlayPlatformId, SourceType, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 7 4 import { LastfmSourceConfig } from "../common/infrastructure/config/source/lastfm.js"; 8 5 import LastfmApiClient, { formatPlayObj } from "../common/vendor/LastfmApiClient.js"; 9 6 import { sortByOldestPlayDate } from "../utils.js"; ··· 12 9 import { Logger } from "@foxxmd/logging"; 13 10 import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; 14 11 import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; 12 12 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 15 13 16 14 export default class LastfmSource extends MemorySource { 17 15 ··· 19 17 requiresAuth = true; 20 18 requiresAuthInteraction = true; 21 19 upstreamType: string = 'Last.fm'; 20 20 + getScrobblesForTimeRange: TimeRangeListensFetcher 22 21 23 22 declare config: LastfmSourceConfig; 24 23 ··· 39 38 this.playerSourceOfTruth = SOURCE_SOT.HISTORY; 40 39 // https://www.last.fm/api/show/user.getRecentTracks 41 40 this.SCROBBLE_BACKLOG_COUNT = 200; 42 42 - this.logger.info(`Note: The player for this source is an analogue for the 'Now Playing' status exposed by ${this.type} which is NOT used for scrobbling. Instead, the 'recently played' or 'history' information provided by this source is used for scrobbles.`) 41 41 + this.logger.info(`Note: The player for this source is an analogue for the 'Now Playing' status exposed by ${this.type} which is NOT used for scrobbling. Instead, the 'recently played' or 'history' information provided by this source is used for scrobbles.`); 42 42 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 43 43 } 44 44 45 45 static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}): PlayObject { ··· 71 71 getLastfmRecentTrack = async(options: RecentlyPlayedOptions = {}): Promise<[PlayObject[], PlayObject[]]> => { 72 72 const {limit = 20} = options; 73 73 try { 74 74 - const plays = await this.api.getRecentTracks({limit}); 75 75 - plays.sort(sortByOldestPlayDate); 74 74 + const {data: plays} = await this.api.getPaginatedTimeRangeListens({limit, cursor: 1}, {includeNowPlaying: true}); 75 75 + const mappedPlayed: PlayObject[] = plays.map(x => ({...x, meta: {...x.meta, sourceSOT: SOURCE_SOT.HISTORY}})); 76 76 + mappedPlayed.sort(sortByOldestPlayDate); 76 77 // if the track is "now playing" it doesn't get a timestamp so we can't determine when it started playing 77 78 // and don't want to accidentally count the same track at different timestamps by artificially assigning it 'now' as a timestamp 78 79 // so we'll just ignore it in the context of recent tracks since really we only want "tracks that have already finished being played" anyway 79 79 - const history = plays.filter(x => x.meta.nowPlaying !== true); 80 80 - const now = plays.filter(x => x.meta.nowPlaying === true); 80 80 + const history = mappedPlayed.filter(x => x.meta.nowPlaying !== true); 81 81 + const now = mappedPlayed.filter(x => x.meta.nowPlaying === true); 81 82 return [history, now]; 82 83 } catch (e) { 83 84 throw e;

+19 -12

src/backend/sources/ListenbrainzSource.ts

Reviewed

··· 1 1 import EventEmitter from "events"; 2 2 - import request from "superagent"; 3 2 import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; 4 3 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 5 5 - import { FormatPlayObjectOptions, InternalConfig, PlayPlatformId } from "../common/infrastructure/Atomic.js"; 4 4 + import { FormatPlayObjectOptions, InternalConfig, PlayPlatformId, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 6 5 import { ListenBrainzSourceConfig } from "../common/infrastructure/config/source/listenbrainz.js"; 7 6 import { ListenbrainzApiClient } from "../common/vendor/ListenbrainzApiClient.js"; 8 7 import { RecentlyPlayedOptions } from "./AbstractSource.js"; ··· 11 10 import { Logger } from "@foxxmd/logging"; 12 11 import { PlayerStateOptions } from "./PlayerState/AbstractPlayerState.js"; 13 12 import { NowPlayingPlayerState } from "./PlayerState/NowPlayingPlayerState.js"; 13 13 + import { ManipulateType } from "dayjs"; 14 14 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 14 15 15 16 export default class ListenbrainzSource extends MemorySource { 16 17 17 18 api: ListenbrainzApiClient; 18 19 requiresAuth = true; 19 20 requiresAuthInteraction = false; 20 20 - isKoito: boolean = false; 21 21 - 21 21 + getScrobblesForTimeRange: TimeRangeListensFetcher 22 22 declare config: ListenBrainzSourceConfig; 23 23 24 24 constructor(name: any, config: ListenBrainzSourceConfig, internal: InternalConfig, emitter: EventEmitter) { ··· 39 39 // 1000 is way too high. maxing at 100 40 40 this.SCROBBLE_BACKLOG_COUNT = 100; 41 41 this.logger.info(`Note: The player for this source is an analogue for the 'Now Playing' status exposed by ${this.type} which is NOT used for scrobbling. Instead, the 'recently played' or 'history' information provided by this source is used for scrobbles.`) 42 42 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this.api, this.api.logger); 42 43 } 43 44 44 44 - static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}){ return ListenbrainzApiClient.formatPlayObj(obj, options); } 45 45 + static formatPlayObj(obj: any, options: FormatPlayObjectOptions = {}){ 46 46 + const play = ListenbrainzApiClient.formatPlayObj(obj, options); 47 47 + play.meta.sourceSOT = SOURCE_SOT.HISTORY; 48 48 + return play; 49 49 + } 45 50 46 51 protected async doCheckConnection(): Promise<true | string | undefined> { 47 52 try { 48 53 await isPortReachableConnect(this.api.url.port, {host: this.api.url.url.hostname}); 49 54 const isKoito = await this.api.checkKoito(); 50 50 - this.api.isKoito = isKoito; 51 51 - this.isKoito = isKoito; 55 55 + if(isKoito) { 56 56 + throw new Error('The given URL looks like a Koito instance. Use the Koito Source instead of Listenbrainz.') 57 57 + } 52 58 return true; 53 59 } catch (e) { 54 60 if(isNodeNetworkException(e)) { ··· 75 81 76 82 getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { 77 83 const {limit = 20} = options; 78 78 - if(this.isKoito) { 79 79 - return await this.api.getRecentlyPlayedKoito(limit); 80 80 - } 81 84 const now = await this.api.getPlayingNow(); 82 85 await this.processRecentPlays(now.listens.map(x => ListenbrainzSource.formatPlayObj(x))); 83 83 - return await this.api.getRecentlyPlayed(limit); 86 86 + return await this.getScrobblesForTimeRange({limit}); 84 87 } 85 88 86 89 getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => { 87 90 try { 88 88 - return await this.api.getRecentlyPlayed(20); 91 91 + return await this.getScrobblesForTimeRange({limit: 20}); 89 92 } catch (e) { 90 93 throw e; 91 94 } 95 95 + } 96 96 + 97 97 + getPaginatedUnitOfTime(): ManipulateType { 98 98 + return 'second'; 92 99 } 93 100 94 101 protected getBackloggedPlays = async (options: RecentlyPlayedOptions = {}) => await this.getRecentlyPlayed({formatted: true, ...options})

+5 -3

src/backend/sources/MalojaSource.ts

Reviewed

··· 1 1 import EventEmitter from "events"; 2 2 import { PlayObject, SOURCE_SOT } from "../../core/Atomic.js"; 3 3 import { isNodeNetworkException } from "../common/errors/NodeErrors.js"; 4 4 - import { InternalConfig } from "../common/infrastructure/Atomic.js"; 4 4 + import { InternalConfig, PaginatedListensTimeRangeOptions, PaginatedTimeRangeListens } from "../common/infrastructure/Atomic.js"; 5 5 import { RecentlyPlayedOptions } from "./AbstractSource.js"; 6 6 import MemorySource from "./MemorySource.js"; 7 7 import { MalojaApiClient } from "../common/vendor/maloja/MalojaApiClient.js"; ··· 56 56 getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => { 57 57 const { limit = 20 } = options; 58 58 await this.processRecentPlays([]); 59 59 - return await this.api.getRecentScrobbles(limit); 59 59 + const resp = await this.api.getPaginatedTimeRangeListens({limit, cursor: 0}); 60 60 + return resp.data; 60 61 } 61 62 62 63 getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => { 63 64 try { 64 64 - return await this.api.getRecentScrobbles(20); 65 65 + const resp = await this.api.getPaginatedTimeRangeListens({limit: 20, cursor: 0}); 66 66 + return resp.data; 65 67 } catch (e) { 66 68 throw e; 67 69 }

+70 -6

src/backend/sources/SpotifySource.ts

Reviewed

··· 12 12 InternalConfig, 13 13 NO_DEVICE, 14 14 NO_USER, 15 15 + PaginatedListensTimeRangeOptions, 16 16 + PaginatedTimeRangeListens, 17 17 + PaginatedTimeRangeListensResult, 15 18 PlayerStateData, 16 19 ReportedPlayerStatus, 17 20 SourceData, 21 21 + TimeRangeListensFetcher, 18 22 } from "../common/infrastructure/Atomic.js"; 19 23 import { SpotifySourceConfig } from "../common/infrastructure/config/source/spotify.js"; 20 24 import { ··· 35 39 import UserDevice = SpotifyApi.UserDevice; 36 40 import { MemoryPositionalSource } from "./MemoryPositionalSource.js"; 37 41 import { baseFormatPlayObj } from "../utils/PlayTransformUtils.js"; 42 42 + import { metaFromJSON } from "@lukehagar/plexjs/sdk/models/operations/getrecentlyadded.js"; 43 43 + import { createGetScrobblesForTimeRangeFunc } from "../utils/ListenFetchUtils.js"; 38 44 39 45 const scopes = ['user-read-recently-played', 'user-read-currently-playing', 'user-read-playback-state', 'user-read-playback-position']; 40 46 const state = 'random'; 41 47 42 48 const shortDeviceId = truncateStringToLength(10, ''); 43 49 44 44 - export default class SpotifySource extends MemoryPositionalSource { 50 50 + export default class SpotifySource extends MemoryPositionalSource implements PaginatedTimeRangeListens<string> { 45 51 46 52 spotifyApi: SpotifyWebApi; 47 53 workingCredsPath: string; ··· 52 58 canGetState = false; 53 59 54 60 declare config: SpotifySourceConfig; 61 61 + 62 62 + getScrobblesForTimeRange: TimeRangeListensFetcher 55 63 56 64 constructor(name: any, config: SpotifySourceConfig, internal: InternalConfig, emitter: EventEmitter) { 57 65 super('spotify', name, config, internal, emitter); ··· 71 79 this.supportsUpstreamRecentlyPlayed = true; 72 80 // https://developer.spotify.com/documentation/web-api/reference/get-recently-played 73 81 this.SCROBBLE_BACKLOG_COUNT = 50 82 82 + this.getScrobblesForTimeRange = createGetScrobblesForTimeRangeFunc(this, this.logger); 74 83 } 75 84 76 85 static formatPlayObj(obj: PlayHistoryObject | CurrentlyPlayingObject, options: FormatPlayObjectOptions = {}): PlayObject { ··· 402 411 return newPlays.map(x => ({...x, meta: {...x.meta, scrobbleTsSOC: SCROBBLE_TS_SOC_END}})) 403 412 } 404 413 414 414 + getPaginatedUnitOfTime(): dayjs.ManipulateType { 415 415 + return 'second'; 416 416 + } 417 417 + 418 418 + getPaginatedTimeRangeListens = async (params: PaginatedListensTimeRangeOptions<string>): Promise<PaginatedTimeRangeListensResult<string>> => { 419 419 + 420 420 + // spotify expects before/after to be unix timestamps WITH millseconds, as number 421 421 + // https://developer.spotify.com/documentation/web-api/reference/get-recently-played 422 422 + const options: Parameters<typeof this.spotifyApi.getMyRecentlyPlayedTracks>[0] = { 423 423 + limit: params.limit, 424 424 + after: undefined 425 425 + }; 426 426 + const { 427 427 + from, 428 428 + to, 429 429 + cursor 430 430 + } = params; 431 431 + 432 432 + if(cursor !== undefined) { 433 433 + options.after = parseInt(cursor, 10); 434 434 + } else if(from !== undefined) { 435 435 + options.after = from * 1000; 436 436 + } 437 437 + 438 438 + try { 439 439 + const result = await this.callApi<ReturnType<typeof this.spotifyApi.getMyRecentlyPlayedTracks>>((api: SpotifyWebApi) => api.getMyRecentlyPlayedTracks(options)); 440 440 + 441 441 + let more = true; 442 442 + let plays = result.body.items.map((x: PlayHistoryObject) => SpotifySource.formatPlayObj(x)).sort(sortByOldestPlayDate); 443 443 + 444 444 + if(to !== undefined) { 445 445 + const toDate = dayjs.unix(to); 446 446 + plays = plays.filter(x => x.data.playDate.isBefore(toDate)); 447 447 + } 448 448 + // if no plays returned 449 449 + // or if filtered plays are less than results then we've hit the to date 450 450 + if(plays.length === 0 || plays.length < result.body.items.length) { 451 451 + more = false; 452 452 + } 453 453 + // failsafe, check the body for next values 454 454 + if(more && result.body.next === null || result.body.cursors === null) { 455 455 + more = false; 456 456 + } 457 457 + 458 458 + return { 459 459 + data: result.body.items.map((x: PlayHistoryObject) => SpotifySource.formatPlayObj(x)).sort(sortByOldestPlayDate), 460 460 + meta: { 461 461 + ...params, 462 462 + total: result.body.total, 463 463 + limit: result.body.limit, 464 464 + more, 465 465 + cursorNext: more && result.body.cursors !== null ? result.body.cursors.after : undefined 466 466 + } 467 467 + } 468 468 + } catch (e) { 469 469 + throw new Error('Error occurred while getting Spotify paginated listens', { cause: e }); 470 470 + } 471 471 + } 472 472 + 405 473 getPlayHistory = async (options: RecentlyPlayedOptions = {}) => { 406 474 const {limit = 20} = options; 407 407 - const func = (api: SpotifyWebApi) => api.getMyRecentlyPlayedTracks({ 408 408 - limit 409 409 - }); 410 410 - const result = await this.callApi<ReturnType<typeof this.spotifyApi.getMyRecentlyPlayedTracks>>(func); 411 411 - return result.body.items.map((x: PlayHistoryObject) => SpotifySource.formatPlayObj(x)).sort(sortByOldestPlayDate); 475 475 + return await this.getScrobblesForTimeRange({limit}); 412 476 } 413 477 414 478 getUpstreamRecentlyPlayed = async (options: RecentlyPlayedOptions = {}): Promise<PlayObject[]> => {

+21 -17

src/backend/sources/SubsonicSource.ts

Reviewed

··· 17 17 import { PlayerStateOptions } from './PlayerState/AbstractPlayerState.js'; 18 18 import { Logger } from '@foxxmd/logging'; 19 19 import { baseFormatPlayObj } from '../utils/PlayTransformUtils.js'; 20 20 + import { noRetryOnUpstreamError, tryApiCall } from '../utils/RequestUtils.js'; 20 21 21 22 dayjs.extend(isSameOrAfter); 22 23 ··· 98 99 return baseFormatPlayObj(obj, play); 99 100 } 100 101 101 101 - callApi = async <T extends SubsonicResponseCommon = SubsonicResponseCommon>(req: Request, retries = 0): Promise<T> => { 102 102 + doCallApi = async <T extends SubsonicResponseCommon = SubsonicResponseCommon>(req: Request, retries = 0): Promise<T> => { 102 103 const { 103 104 data: { 104 105 user, ··· 175 176 } else { 176 177 this.logger.error(`${tokenError} Your config settings do not allow legacy authentication to be used.`); 177 178 } 179 179 + uError.showStopper = true; 178 180 throw uError; 179 181 } else { 180 180 - this.logger.warn(`${parseApiResponseErrorToThrowable(resp)} | ${tokenError} MS will attempt to use legacy authentication since 'legacyAuthentication' is not explicitly defined (or disabled) in config.`); 181 182 this.config.data.legacyAuthentication = true; 182 182 - return await this.callApi(req); 183 183 + throw new UpstreamError(`${tokenError} MS will attempt to use legacy authentication since 'legacyAuthentication' is not explicitly defined (or disabled) in config.`, {cause: uError, showStopper: false}); 183 184 } 185 185 + } else { 186 186 + throw uError; 184 187 } 185 188 } 186 189 ··· 191 194 throw e; 192 195 } 193 196 194 194 - if((isNodeNetworkException(e) || e.status >= 500) && retries < maxRequestRetries) { 195 195 - const retryAfter = parseRetryAfterSecsFromObj(e) ?? (retryMultiplier * (retries + 1)); 196 196 - this.logger.warn(`Request failed but retries (${retries}) less than max (${maxRequestRetries}), retrying request after ${retryAfter} seconds...`); 197 197 - await sleep(retryAfter * 1000); 198 198 - return await this.callApi(req, retries + 1) 199 199 - } 200 200 - 201 201 - if(isNodeNetworkException(e)) { 202 202 - throw new UpstreamError('Could not communicate with Subsonic Server', {cause: e, showStopper: true}); 203 203 - } 204 204 - 205 197 if(e.message.includes('self-signed certificate')) { 206 198 throw new UpstreamError(`Subsonic server uses self-signed certs which MS does not allow by default. This error can be ignored by setting 'ignoreTlsErrors: true' in config. WARNING this can result in cleartext communication which is insecure.`, {cause: e, showStopper: true}); 207 199 } ··· 210 202 } 211 203 } 212 204 205 205 + callApi = async <T extends SubsonicResponseCommon = SubsonicResponseCommon>(reqFunc: () => Request): Promise<T> => { 206 206 + try { 207 207 + return await tryApiCall(() => this.doCallApi(reqFunc()), { 208 208 + ...this.config, 209 209 + logger: this.logger, 210 210 + shouldRetry: noRetryOnUpstreamError 211 211 + }) as T; 212 212 + } catch (e) { 213 213 + throw e; 214 214 + } 215 215 + } 216 216 + 213 217 protected async doBuildInitData(): Promise<true | string | undefined> { 214 218 const {data: {user, password, url} = {}} = this.config; 215 219 ··· 245 249 protected async doCheckConnection(): Promise<true | string | undefined> { 246 250 const {url} = this.config.data; 247 251 try { 248 248 - const resp = await this.callApi(request.get(`${url}/rest/ping`)); 252 252 + const resp = await this.callApi(() => request.get(`${url}/rest/ping`)); 249 253 this.sourceData = resp as SourceIdentifierData; 250 254 this.logger.info(`Subsonic Server reachable: ${identifiersFromResponse(resp)}`); 251 255 return true; ··· 274 278 doAuthentication = async () => { 275 279 const {url} = this.config.data; 276 280 try { 277 277 - await this.callApi(request.get(`${url}/rest/ping`)); 281 281 + await this.callApi(() => request.get(`${url}/rest/ping`)); 278 282 this.logger.info('Subsonic API Status: ok'); 279 283 return true; 280 284 } catch (e) { ··· 285 289 getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => { 286 290 const {formatted = false} = options; 287 291 const {url} = this.config.data; 288 288 - const resp = await this.callApi(request.get(`${url}/rest/getNowPlaying`)); 292 292 + const resp = await this.callApi(() => request.get(`${url}/rest/getNowPlaying`)); 289 293 const { 290 294 nowPlaying: { 291 295 entry = []

+3 -4

src/backend/sources/TealfmSource.ts

Reviewed

··· 8 8 import { TealSourceConfig } from "../common/infrastructure/config/source/tealfm.js"; 9 9 import { BlueSkyAppApiClient } from "../common/vendor/bluesky/BlueSkyAppApiClient.js"; 10 10 import { BlueSkyOauthApiClient } from "../common/vendor/bluesky/BlueSkyOauthApiClient.js"; 11 11 - import { ListRecord, ScrobbleRecord } from "../common/infrastructure/config/client/tealfm.js"; 12 11 import { parseArrayFromMaybeString } from "../utils/StringUtils.js"; 13 12 14 13 export default class TealfmSource extends MemorySource { ··· 107 106 108 107 getRecentlyPlayed = async(options: RecentlyPlayedOptions = {}) => { 109 108 const {limit = 20} = options; 110 110 - let list: ListRecord<ScrobbleRecord>[]; 109 109 + let plays: PlayObject[]; 111 110 try { 112 112 - list = await this.client.listScrobbleRecord(limit) 111 111 + const {data} = await this.client.getPagelessTimeRangeListens({limit}) 112 112 + plays = data; 113 113 } catch (e) { 114 114 throw new Error('Error occurred while trying to fetch records', {cause: e}); 115 115 } 116 116 await this.processRecentPlays([]); 117 117 - let plays = list.map(x => listRecordToPlay(x)); 118 117 if(this.serviceAllow.length > 0) { 119 118 plays = plays.filter(x => 120 119 (x.meta.musicService !== undefined && this.serviceAllow.some(y => x.meta.musicService.toLocaleLowerCase().includes(y)))

+1

src/backend/sources/WebScrobblerSource.ts

Reviewed

··· 85 85 } = obj; 86 86 87 87 const play = WebScrobblerSource.formatPlayObj(obj.data.song, {nowPlaying: eventName !== 'scrobble'}); 88 88 + play.meta.sourceSOT = SOURCE_SOT.HISTORY; 88 89 return { 89 90 platformId: [play.meta.deviceId, NO_USER], 90 91 play,

+42 -1

src/backend/tests/listenbrainz/listenbrainz.test.ts

Reviewed

··· 1 1 - import { loggerTest } from "@foxxmd/logging"; 1 1 + import { loggerDebug, loggerTest } from "@foxxmd/logging"; 2 2 import { assert, expect } from 'chai'; 3 3 import dayjs from "dayjs"; 4 4 import { describe, it } from 'mocha'; ··· 154 154 } 155 155 } 156 156 )); 157 157 + 158 158 + it('Should retry non bad-requests',async function () { 159 159 + await withRequestInterception( 160 160 + [ 161 161 + http.post('https://api.listenbrainz.org/1/submit-listens', function* () { 162 162 + let hit = false; 163 163 + 164 164 + while(!hit) { 165 165 + hit = true; 166 166 + yield HttpResponse.text('Gateway', {status: 503}); 167 167 + } 168 168 + return HttpResponse.text('OK', {status: 200}); 169 169 + }) 170 170 + ], 171 171 + async function() { 172 172 + const play: PlayObject = { 173 173 + data: { 174 174 + artists: ['Celldweller'], 175 175 + album: 'The Complete Cellout, Volume 01', 176 176 + track: 'Frozen', 177 177 + duration: 299, 178 178 + playDate: dayjs(), 179 179 + meta: { 180 180 + brainz: { 181 181 + // @ts-expect-error wrong on purpose 182 182 + artist: 'fad8967c-a327-4af5-a64a-d4de66ece652;100846a7-06f6-4129-97ce-4409b9a9a311', 183 183 + album: '2eb6a8fb-14f6-436e-9bdf-2f9d0d8cbae0', 184 184 + recording: '677862e0-3603-4120-8c44-ee9a70893647', 185 185 + releaseGroup: 'bd3bb964-6da7-4d59-b0aa-f8bf639cd419' 186 186 + } 187 187 + } 188 188 + }, 189 189 + meta: { 190 190 + lifecycle: defaultLifecycle() 191 191 + } 192 192 + } 193 193 + const res = await client.submitListen(play); 194 194 + expect(res.response).eq('OK'); 195 195 + } 196 196 + )(); 197 197 + }) 157 198 }); 158 199 159 200 describe('Listenbrainz Endpoint Behavior', function() {

+5 -4

src/backend/tests/scrobbler/TestScrobbler.ts

Reviewed

··· 6 6 import AbstractScrobbleClient from "../../scrobblers/AbstractScrobbleClient.js"; 7 7 import { CommonClientConfig, CommonClientOptions, NowPlayingOptions } from "../../common/infrastructure/config/client/index.js"; 8 8 import clone from "clone"; 9 9 + import { TimeRangeListensFetcher } from "../../common/infrastructure/Atomic.js"; 9 10 10 11 export class TestScrobbler extends AbstractScrobbleClient { 11 12 12 13 testRecentScrobbles: PlayObject[] = []; 14 14 + getScrobblesForTimeRange: TimeRangeListensFetcher; 13 15 14 16 constructor(config: CommonClientConfig = {name: 'test'}) { 15 17 const logger = loggerTest; 16 18 const notifier = new Notifiers(new EventEmitter(), new EventEmitter(), new EventEmitter(), logger); 17 19 super('test', 'Test', {name: 'test', ...config}, notifier, new EventEmitter(), logger); 18 20 this.supportsNowPlaying = false; 19 19 - } 20 20 - 21 21 - protected async getScrobblesForRefresh(limit: number): Promise<PlayObject[]> { 22 22 - return this.testRecentScrobbles; 21 21 + this.getScrobblesForTimeRange = async (_) => this.testRecentScrobbles; 22 22 + this.scrobbleDelay = 10; 23 23 + this.scrobbleSleep = 20; 23 24 } 24 25 25 26 doScrobble(playObj: PlayObject) {

+187 -82

src/backend/tests/scrobbler/scrobblers.test.ts

Reviewed

··· 1 1 import chai, { assert, expect } from 'chai'; 2 2 + import {spy} from 'sinon'; 2 3 import asPromised from 'chai-as-promised'; 3 4 import clone from 'clone'; 4 4 - import { source } from "common-tags"; 5 5 import dayjs from "dayjs"; 6 6 import { after, before, describe, it } from 'mocha'; 7 7 import { http, HttpResponse } from 'msw'; 8 8 import pEvent from 'p-event'; 9 9 - import { PlayObject } from "../../../core/Atomic.js"; 10 10 - import { genGroupIdStr, sleep } from "../../utils.js"; 9 9 + import { PlayObject, SOURCE_SOT } from "../../../core/Atomic.js"; 10 10 + import { genGroupIdStr, sleep, sortByOldestPlayDate } from "../../utils.js"; 11 11 import mixedDuration from '../plays/mixedDuration.json' with { type: 'json' }; 12 12 import withDuration from '../plays/withDuration.json' with { type: 'json' }; 13 13 import { MockNetworkError, withRequestInterception } from "../utils/networking.js"; ··· 15 15 import MockDate from 'mockdate'; 16 16 17 17 import { NowPlayingScrobbler, TestAuthScrobbler, TestScrobbler } from "./TestScrobbler.js"; 18 18 - import { PlayPlatformId } from '../../common/infrastructure/Atomic.js'; 18 18 + import { PaginatedTimeRangeOptions, PlayPlatformId, REFRESH_STALE_DEFAULT } from '../../common/infrastructure/Atomic.js'; 19 19 import { defaultLifecycle } from '../../utils/PlayTransformUtils.js'; 20 20 + import { shuffleArray } from '../../utils/DataUtils.js'; 21 21 + import { DEFAULT_CONSOLIDATE_DURATION, DEFAULT_GROUP_DURATION, groupPlaysToTimeRanges } from '../../utils/ListenFetchUtils.js'; 20 22 21 23 chai.use(asPromised); 22 24 ··· 39 41 confidenceBreakdown: true 40 42 } 41 43 }; 42 42 - testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); 43 44 return testScrobbler; 44 45 } 45 46 ··· 117 118 118 119 it('It is not detected as duplicate when play date is newer than most recent', async function () { 119 120 120 120 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 121 121 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 121 122 122 123 const newScrobble = generatePlay({ 123 124 playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 1].data.playDate.add(70, 'seconds') ··· 129 130 130 131 it('It is not detected as duplicate when play date is close to an existing scrobble', async function () { 131 132 132 132 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 133 133 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 133 134 134 135 const newScrobble = generatePlay({ 135 136 playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds') ··· 140 141 141 142 it('It handles unique detection when no existing scrobble matches above a score of 0', async function () { 142 143 143 143 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 144 144 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 144 145 145 146 const uniquePlay = generatePlay({ 146 147 artists: [ ··· 164 165 165 166 it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () { 166 167 167 167 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 168 168 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 168 169 169 170 const diffPlay = clone(normalizedWithMixedDur[1]); 170 171 diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's'); ··· 175 176 176 177 it('Is not detected as duplicate when track is same, time is similar, but artist is different', async function () { 177 178 178 178 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 179 179 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 179 180 180 181 const diffPlay = clone(normalizedWithMixedDur[1]); 181 182 diffPlay.data.playDate = diffPlay.data.playDate.add(9, 's'); ··· 187 188 188 189 it('Is not detected as duplicate when play date is different by more than 10 seconds (high granularity source)', async function () { 189 190 190 190 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 191 191 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 191 192 192 193 const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]); 193 194 timeOffPos.data.playDate = timeOffPos.data.playDate.add(11, 's'); ··· 205 206 initialDate: firstPlayDate, 206 207 defaultMeta: {source: 'subsonic'} 207 208 }); 208 208 - testScrobbler.recentScrobbles = recent; 209 209 + testScrobbler.testRecentScrobbles = recent; 209 210 210 211 const timeOffPos = clone(recent[recent.length - 1]); 211 212 timeOffPos.data.playDate = timeOffPos.data.playDate.add(61, 's'); ··· 225 226 226 227 it('A track with continuity to the previous track is not detected as a duplicate', async function () { 227 228 228 228 - testScrobbler.recentScrobbles = normalizedWithDur; 229 229 + testScrobbler.testRecentScrobbles = normalizedWithDur; 229 230 230 231 const brickPt1 = normalizedWithDur.find(x => x.data.track.includes('Another Brick')); 231 232 const brickPt2 = clone(brickPt1); ··· 254 255 initialDate: firstPlayDate, 255 256 defaultMeta: {source: 'jellyfin'} 256 257 }); 257 257 - testScrobbler.recentScrobbles = recent; 258 258 + testScrobbler.testRecentScrobbles = recent; 258 259 259 260 const repeatPlay = clone(recent[recent.length - 1]); 260 261 repeatPlay.data.playDate = repeatPlay.data.playDate.add(repeatPlay.data.duration + 2, 's'); ··· 264 265 repeatPlay.data.repeat = true; 265 266 assert.isFalse((await testScrobbler.alreadyScrobbled(repeatPlay))[0]); 266 267 }); 268 268 + 269 269 + it('Is not detected as duplicate when play date matches fuzzy and play source SOT is history', async function () { 270 270 + 271 271 + const play = generatePlay({ 272 272 + artists: ['Nejad'], 273 273 + track: 'CODE', 274 274 + album: undefined, 275 275 + playDate: dayjs().subtract(179, 's'), 276 276 + duration: 179 277 277 + }); 278 278 + testScrobbler.testRecentScrobbles = [play]; 279 279 + 280 280 + const newPlay = clone(play); 281 281 + newPlay.data.playDate = dayjs(); 282 282 + newPlay.meta.sourceSOT = SOURCE_SOT.HISTORY; 283 283 + 284 284 + const res = await testScrobbler.existingScrobble(newPlay, [play]); 285 285 + expect(res.match).is.false; 286 286 + }); 267 287 }); 268 288 269 289 }); ··· 275 295 }); 276 296 277 297 it('Is detected as duplicate when an exact match', async function () { 278 278 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 298 298 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 279 299 assert.isTrue((await testScrobbler.alreadyScrobbled(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]))[0]); 280 300 }); 281 301 282 302 it('Is detected as duplicate when artist/title differences are whitespace or case', async function () { 283 283 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 303 303 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 284 304 const ref = normalizedWithMixedDur[3]; 285 305 286 306 const diffPlay = clone(ref); ··· 304 324 }); 305 325 306 326 it('Is detected as duplicate when artist/title differences are from unicode normalization', async function () { 307 307 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 327 327 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 308 328 const ref = normalizedWithMixedDur.find(x => x.data.track === 'Jimbó'); 309 329 310 330 const diffPlay = clone(ref); ··· 315 335 316 336 it('Is detected as duplicate when play date is off by 10 seconds or less (high granularity source)', async function () { 317 337 318 318 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 338 338 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 319 339 320 340 const timeOffPos = clone(normalizedWithMixedDur[normalizedWithMixedDur.length - 1]); 321 341 timeOffPos.data.playDate = timeOffPos.data.playDate.add(10, 's'); ··· 331 351 son.data.playDate = dayjs().subtract(1, 'hour').set('minute', 26).set('second', 20); 332 352 son.data.duration = 267; 333 353 son.data.listenedFor = undefined; 334 334 - testScrobbler.recentScrobbles = normalizedWithMixedDurOlder.concat(son); 354 354 + testScrobbler.testRecentScrobbles = normalizedWithMixedDurOlder.concat(son); 335 355 336 356 const offSon = clone(son); 337 357 offSon.data.playDate = dayjs().subtract(1, 'hour').set('minute', 30).set('second', 37); ··· 344 364 initialDate: firstPlayDate, 345 365 defaultMeta: {source: 'subsonic'} 346 366 }); 347 347 - testScrobbler.recentScrobbles = recent; 367 367 + testScrobbler.testRecentScrobbles = recent; 348 368 349 369 const timeOffPos = clone(recent[recent.length - 1]); 350 370 timeOffPos.data.playDate = timeOffPos.data.playDate.add(59, 's'); ··· 357 377 }); 358 378 359 379 it('Is detected as duplicate when title is exact, artist is similar, and time is similar', async function () { 360 360 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 380 380 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 361 381 const ref = normalizedWithMixedDur[3]; 362 382 363 383 const diffPlay = clone(ref); ··· 402 422 } 403 423 } 404 424 405 405 - testScrobbler.recentScrobbles = normalizedWithMixedDurOlder.concat(ref); 425 425 + testScrobbler.testRecentScrobbles = normalizedWithMixedDurOlder.concat(ref); 406 426 407 427 assert.isTrue((await testScrobbler.alreadyScrobbled(spotifyPlay))[0]); 408 428 }); ··· 415 435 416 436 it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () { 417 437 418 418 - testScrobbler.recentScrobbles = normalizedWithDur; 438 438 + testScrobbler.testRecentScrobbles = normalizedWithDur; 419 439 420 440 const timeEnd = clone(normalizedWithDur[normalizedWithMixedDur.length - 2]); 421 441 timeEnd.data.playDate = timeEnd.data.playDate.add(timeEnd.data.duration, 's'); ··· 441 461 442 462 beforeEach(function() { 443 463 testScrobbler = generateTestScrobbler(); 444 444 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 445 445 - testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); 464 464 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 446 465 }); 447 447 - 448 448 - // before(function () { 449 449 - // testScrobbler.recentScrobbles = normalizedWithMixedDur; 450 450 - // testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); 451 451 - // }); 452 466 453 467 it('Detects a unique play', async function() { 454 468 const newScrobble = generatePlay({ ··· 491 505 492 506 describe('Upstream Scrobbles', function() { 493 507 494 494 - it('Stores upstream scrobbles on refresh', async function () { 495 495 - const scrobbler = generateTestScrobbler(); 496 496 - scrobbler.testRecentScrobbles = normalizedWithMixedDur; 497 497 - assert.isEmpty(scrobbler.recentScrobbles); 498 498 - await scrobbler.refreshScrobbles(); 499 499 - assert.isNotEmpty(scrobbler.recentScrobbles); 508 508 + afterEach(function () { 509 509 + MockDate.reset(); 500 510 }); 501 511 502 502 - describe('Detects when upstream scrobbles should be refreshed', function() { 512 512 + it('Calls timerange func to get SOT scrobbles when none exists', async function() { 513 513 + const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 514 514 + const scrobbler = generateTestScrobbler(); 515 515 + scrobbler.testRecentScrobbles = []; 516 516 + await scrobbler.tryInitialize(); 517 517 + scrobbler.testRecentScrobbles = existingPlays; 503 518 504 504 - const normalizedClose = normalizePlays(generatePlays(10), {endDate: dayjs().subtract(100, 'seconds')}); 519 519 + const sp = spy(scrobbler, 'getScrobblesForTimeRange'); 505 520 506 506 - beforeEach(async function () { 507 507 - testScrobbler = generateTestScrobbler(); 508 508 - testScrobbler.testRecentScrobbles = normalizedClose; 509 509 - await testScrobbler.initialize(); 510 510 - testScrobbler.lastScrobbleCheck = dayjs().subtract(65, 'seconds'); 511 511 - testScrobbler.queuedScrobbles = []; 512 512 - testScrobbler.config.options = {}; 513 513 - }); 521 521 + const play = generatePlay({playDate: dayjs().subtract(60, 's')}); 522 522 + await scrobbler.queueScrobble(play, 'test'); 523 523 + const emptied = pEvent(scrobbler.emitter, 'queueEmptied'); 524 524 + scrobbler.startScrobbling().then(() => null); 525 525 + await emptied; 526 526 + scrobbler.tryStopScrobbling().then(() => null); 527 527 + expect(sp.called).is.true; 528 528 + }); 529 529 + 530 530 + it('Uses cached timerange for closely grouped scrobbles', async function() { 531 531 + const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 532 532 + const scrobbler = generateTestScrobbler(); 533 533 + scrobbler.testRecentScrobbles = []; 534 534 + await scrobbler.tryInitialize(); 535 535 + scrobbler.testRecentScrobbles = existingPlays; 514 536 515 515 - it('Detects queued scrobble date is newer than last scrobble refresh', async function() { 516 516 - const newScrobble = generatePlay({ 517 517 - playDate: dayjs() 518 518 - }); 537 537 + const sp = spy(scrobbler, 'getScrobblesForTimeRange'); 519 538 520 520 - await testScrobbler.queueScrobble(newScrobble, 'test'); 521 521 - assert.isTrue(testScrobbler.shouldRefreshScrobble()); 522 522 - }); 539 539 + const play1 = generatePlay({playDate: dayjs().subtract(3, 'm')}); 540 540 + const play2 = generatePlay({playDate: dayjs().subtract(1, 'm')}); 541 541 + await scrobbler.queueScrobble([play1, play2], 'test'); 542 542 + const emptied = pEvent(scrobbler.emitter, 'queueEmptied'); 543 543 + scrobbler.startScrobbling().then(() => null); 544 544 + await emptied; 545 545 + scrobbler.tryStopScrobbling().then(() => null); 546 546 + expect(sp.calledOnce).is.true; 547 547 + }); 523 548 524 524 - it('Detects queued scrobble date is older than newest scrobble', async function() { 525 525 - const newScrobble = generatePlay({ 526 526 - playDate: dayjs().subtract(120, 'seconds') 527 527 - }); 549 549 + it('Uses separate timerange calls when scrobbles are not closely grouped', async function() { 550 550 + const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 551 551 + const scrobbler = generateTestScrobbler(); 552 552 + scrobbler.testRecentScrobbles = []; 553 553 + await scrobbler.tryInitialize(); 554 554 + scrobbler.testRecentScrobbles = existingPlays; 528 555 529 529 - await testScrobbler.queueScrobble(newScrobble, 'test'); 530 530 - assert.isTrue(testScrobbler.shouldRefreshScrobble()); 531 531 - }); 556 556 + const sp = spy(scrobbler, 'getScrobblesForTimeRange'); 532 557 533 533 - it('Forces refresh if refreshStaleAfter is set', async function() { 534 534 - testScrobbler.config.options = { refreshStaleAfter: 10 }; 558 558 + const play1 = generatePlay({playDate: dayjs().subtract(3, 'm')}); 559 559 + const play2 = generatePlay({playDate: dayjs().subtract(1, 'm')}); 560 560 + const play3 = generatePlay({playDate: dayjs().subtract(DEFAULT_CONSOLIDATE_DURATION.add(4, 'm'))}); 561 561 + await scrobbler.queueScrobble([play1, play2, play3], 'test'); 562 562 + const emptied = pEvent(scrobbler.emitter, 'queueEmptied'); 563 563 + scrobbler.startScrobbling().then(() => null); 564 564 + await emptied; 565 565 + scrobbler.tryStopScrobbling().then(() => null); 566 566 + expect(sp.calledTwice).is.true; 567 567 + }); 535 568 536 536 - const newScrobble = generatePlay({ 537 537 - playDate: dayjs().subtract(80, 'seconds') 538 538 - }); 569 569 + it('Gets fresh timerange if TTL of staleAfter has passed', async function() { 570 570 + const existingPlays = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 571 571 + const scrobbler = generateTestScrobbler(); 572 572 + scrobbler.testRecentScrobbles = []; 573 573 + await scrobbler.tryInitialize(); 574 574 + scrobbler.testRecentScrobbles = existingPlays; 539 575 540 540 - await testScrobbler.queueScrobble(newScrobble, 'test'); 541 541 - assert.isTrue(testScrobbler.shouldRefreshScrobble()); 542 542 - }); 576 576 + const sp = spy(scrobbler, 'getScrobblesForTimeRange'); 543 577 544 544 - it('Does not refresh if scrobble is older than last check but newer than newest upstream scrobble', async function() { 545 545 - testScrobbler.lastScrobbleCheck = dayjs().subtract(40, 'seconds'); 546 546 - const newScrobble = generatePlay({ 547 547 - playDate: dayjs().subtract(80, 'seconds') 548 548 - }); 578 578 + const play1 = generatePlay({playDate: dayjs().subtract(3, 'm')}); 579 579 + const play2 = generatePlay({playDate: dayjs().subtract(1, 'm')}); 580 580 + await scrobbler.queueScrobble([play1], 'test'); 581 581 + const emptied = pEvent(scrobbler.emitter, 'queueEmptied'); 582 582 + scrobbler.startScrobbling().then(() => null); 583 583 + await emptied; 584 584 + expect(sp.calledOnce).is.true; 549 585 550 550 - await testScrobbler.queueScrobble(newScrobble, 'test'); 551 551 - assert.isFalse(testScrobbler.shouldRefreshScrobble()); 552 552 - }); 586 586 + MockDate.set(dayjs().add(REFRESH_STALE_DEFAULT + 1, 's').toDate()); 587 587 + const emptied2 = pEvent(scrobbler.emitter, 'queueEmptied'); 588 588 + await scrobbler.queueScrobble([play2], 'test'); 589 589 + await emptied2; 590 590 + scrobbler.tryStopScrobbling().then(() => null); 591 591 + expect(sp.calledTwice).is.true; 553 592 }); 554 593 555 594 }); ··· 561 600 beforeEach(async function() { 562 601 testScrobbler = generateTestScrobbler(); 563 602 await testScrobbler.initialize(); 564 564 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 603 603 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 565 604 testScrobbler.scrobbleSleep = 500; 566 605 testScrobbler.scrobbleDelay = 0; 567 567 - testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); 606 606 + //testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); 568 607 testScrobbler.config.options = {}; 569 608 //testScrobbler.initScrobbleMonitoring().catch(console.error); 570 609 }); ··· 629 668 track: 'my hugely cool and very different track title' 630 669 }); 631 670 632 632 - testScrobbler.recentScrobbles = normalizePlays([newScrobble, ...withDurPlays], {initialDate: firstPlayDate}); 671 671 + testScrobbler.testRecentScrobbles = normalizePlays([newScrobble, ...withDurPlays], {initialDate: firstPlayDate}); 633 672 testScrobbler.buildTransformRules(); 634 673 635 674 expect((await testScrobbler.alreadyScrobbled(newScrobble))[0]).is.false; ··· 651 690 track: 'my hugely cool and very different track title' 652 691 }); 653 692 654 654 - testScrobbler.recentScrobbles = normalizePlays([newScrobble, ...withDurPlays], {initialDate: firstPlayDate}); 693 693 + testScrobbler.testRecentScrobbles = normalizePlays([newScrobble, ...withDurPlays], {initialDate: firstPlayDate}); 655 694 testScrobbler.buildTransformRules(); 656 695 657 696 expect((await testScrobbler.alreadyScrobbled(newScrobble))[0]).is.false; ··· 669 708 beforeEach(async function() { 670 709 testScrobbler = generateTestScrobbler(); 671 710 await testScrobbler.initialize(); 672 672 - testScrobbler.recentScrobbles = normalizedWithMixedDur; 711 711 + testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 673 712 testScrobbler.testRecentScrobbles = normalizedWithMixedDur; 674 713 testScrobbler.scrobbleSleep = 500; 675 714 testScrobbler.scrobbleDelay = 0; 676 676 - testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); 677 715 testScrobbler.initScrobbleMonitoring().catch(console.error); 678 716 }); 679 717 ··· 993 1031 }); 994 1032 995 1033 }); 1034 1034 + 1035 1035 + describe('Scrobble Temporal Grouping', function () { 1036 1036 + 1037 1037 + it('Groups into separate groups when not within duration', function() { 1038 1038 + const plays1 = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 1039 1039 + plays1.sort(sortByOldestPlayDate); 1040 1040 + const oldest1 = plays1[0].data.playDate.unix(); 1041 1041 + const newest1 = plays1[plays1.length - 1].data.playDate.unix(); 1042 1042 + 1043 1043 + const plays2 = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(2, 'hour')}); 1044 1044 + plays2.sort(sortByOldestPlayDate); 1045 1045 + const oldest2 = plays2[0].data.playDate.unix(); 1046 1046 + const newest2 = plays2[plays2.length - 1].data.playDate.unix(); 1047 1047 + 1048 1048 + const plays = [...plays1, ...plays2]; 1049 1049 + shuffleArray(plays); 1050 1050 + 1051 1051 + const ranges = groupPlaysToTimeRanges(plays, [], {consolidateDuration: dayjs.duration(1, 's')}); 1052 1052 + expect(ranges.length).eq(2); 1053 1053 + expect(ranges.some(x => x.from < oldest1 && x.to > newest1)).is.true; 1054 1054 + expect(ranges.some(x => x.from < oldest2 && x.to > newest2)).is.true; 1055 1055 + }); 1056 1056 + 1057 1057 + it('Groups into existing time range', function() { 1058 1058 + const plays1 = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 1059 1059 + plays1.sort(sortByOldestPlayDate); 1060 1060 + const oldest1 = plays1[0].data.playDate; 1061 1061 + const newest1 = plays1[plays1.length - 1].data.playDate; 1062 1062 + 1063 1063 + const plays2 = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(2, 'hour')}); 1064 1064 + plays2.sort(sortByOldestPlayDate); 1065 1065 + const oldest2 = plays2[0].data.playDate; 1066 1066 + const newest2 = plays2[plays2.length - 1].data.playDate; 1067 1067 + 1068 1068 + const plays = [...plays1, ...plays2]; 1069 1069 + shuffleArray(plays); 1070 1070 + 1071 1071 + const existing: PaginatedTimeRangeOptions = {from: oldest1.subtract(10, 's').unix(), to: newest1.add(10, 's').unix()}; 1072 1072 + 1073 1073 + const ranges = groupPlaysToTimeRanges(plays, [existing], {consolidateDuration: dayjs.duration(1, 's')}); 1074 1074 + expect(ranges.length).eq(2); 1075 1075 + expect(ranges.some(x => x.from < oldest1.unix() && x.to > newest1.unix())).is.true; 1076 1076 + expect(ranges.some(x => x.from < oldest2.unix() && x.to > newest2.unix())).is.true; 1077 1077 + expect(ranges.some(x => x.to === existing.to && x.from && existing.from)); 1078 1078 + }); 1079 1079 + 1080 1080 + it('Consolidates time ranges', function() { 1081 1081 + const plays1 = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(1, 'hour')}); 1082 1082 + plays1.sort(sortByOldestPlayDate); 1083 1083 + 1084 1084 + const plays2 = normalizePlays(generatePlays(3), {initialDate: dayjs().subtract(5, 'hour')}); 1085 1085 + plays2.sort(sortByOldestPlayDate); 1086 1086 + 1087 1087 + const singlePlay1 = generatePlay({playDate: dayjs().subtract(9, 'h')}) 1088 1088 + const singlePlay2 = generatePlay({playDate: dayjs().subtract(9, 'h').subtract(20, 'm')}); 1089 1089 + const singlePlay3 = generatePlay({playDate: dayjs().subtract(9, 'h').subtract(40, 'm')}); 1090 1090 + 1091 1091 + const plays = [...plays1, ...plays2, singlePlay1, singlePlay2, singlePlay3]; 1092 1092 + shuffleArray(plays); 1093 1093 + 1094 1094 + const ranges = groupPlaysToTimeRanges(plays, [], {consolidateDuration: dayjs.duration(1, 's')}); 1095 1095 + expect(ranges.length).eq(5); 1096 1096 + 1097 1097 + const consolidatedRanges = groupPlaysToTimeRanges(plays, [], {consolidateDuration: DEFAULT_CONSOLIDATE_DURATION}); 1098 1098 + expect(consolidatedRanges.length).eq(3); 1099 1099 + }); 1100 1100 + })

+2 -1

src/backend/utils.ts

Reviewed

··· 63 63 return new Promise(resolve => setTimeout(resolve, ms)); 64 64 } 65 65 66 66 - // sorts playObj formatted objects by playDate in ascending (oldest first) order 66 66 + /** sorts playObj formatted objects by playDate in ascending (oldest first) order */ 67 67 export const sortByOldestPlayDate = (a: PlayObject, b: PlayObject) => { 68 68 const { 69 69 data: { ··· 87 87 return aPlayDate.isAfter(bPlayDate) ? 1 : -1 88 88 }; 89 89 90 90 + /** sorts playObj formatted objects by playDate in descending (newest first) order */ 90 91 export const sortByNewestPlayDate = (a: PlayObject, b: PlayObject) => { 91 92 const { 92 93 data: {

+11

src/backend/utils/DataUtils.ts

Reviewed

··· 163 163 } 164 164 165 165 return Object.values(obj).every(x => valueIsEmpty(x)); 166 166 + } 167 167 + 168 168 + /** Shuffle array in-place 169 169 + * 170 170 + * https://stackoverflow.com/a/12646864/1469797 171 171 + */ 172 172 + export const shuffleArray = (array: any[]): void => { 173 173 + for (let i = array.length - 1; i > 0; i--) { 174 174 + const j = Math.floor(Math.random() * (i + 1)); 175 175 + [array[i], array[j]] = [array[j], array[i]]; 176 176 + } 166 177 }

+322

src/backend/utils/ListenFetchUtils.ts

Reviewed

··· 1 1 + import { childLogger, Logger, loggerTest } from "@foxxmd/logging"; 2 2 + import dayjs, { Dayjs } from "dayjs"; 3 3 + import { Duration } from "dayjs/plugin/duration.js"; 4 4 + import { PlayObject, UnixTimestamp } from "../../core/Atomic.js"; 5 5 + import { CursorType, hasPagelessTimeRangeListens, hasPaginatedTimeRangeListens, PagelessListensTimeRangeOptions, PagelessTimeRangeListens, PagelessTimeRangeListensResult, PaginatedListensTimeRangeOptions, PaginatedTimeRangeCommonOptions, PaginatedTimeRangeListens, PaginatedTimeRangeListensResult, PaginatedTimeRangeOptions, PaginatedTimeRangeSource, REFRESH_STALE_DEFAULT, TimeRangeListensFetcher } from "../common/infrastructure/Atomic.js"; 6 6 + import { loggerNoop, MaybeLogger } from "../common/logging.js"; 7 7 + import { sortByNewestPlayDate, sortByOldestPlayDate } from "../utils.js"; 8 8 + import { todayAwareFormat } from "./TimeUtils.js"; 9 9 + import { playDateWithinDurationOfAny } from "./PlayComparisonUtils.js"; 10 10 + 11 11 + export interface TimeRangeFetchOptions { 12 12 + logger?: MaybeLogger | Logger 13 13 + 14 14 + } 15 15 + 16 16 + export const createGetScrobblesForTimeRangeFunc = <T extends PaginatedTimeRangeSource>(fetcher: T, pLogger = loggerNoop): TimeRangeListensFetcher => { 17 17 + let requestCount: number; 18 18 + const logger = childLogger(pLogger, ['Pagination']); 19 19 + const reqLabel = () => `Request ${requestCount}`; 20 20 + const reqLogger = childLogger(logger, [reqLabel]); 21 21 + 22 22 + let plays: PlayObject[] = []; 23 23 + 24 24 + if (hasPagelessTimeRangeListens(fetcher)) { 25 25 + return async (opts: PaginatedTimeRangeCommonOptions): Promise<PlayObject[]> => { 26 26 + requestCount = 0; 27 27 + let more = true; 28 28 + let currOpts = { ...opts }; 29 29 + let initial = true; 30 30 + while (more) { 31 31 + requestCount++; 32 32 + const reqOptsHint: string[] = [ 33 33 + `Between ${todayAwareFormat(dayjs(currOpts.from))} and ${todayAwareFormat(dayjs(currOpts.to))}` 34 34 + ]; 35 35 + if(currOpts.to !== undefined && currOpts.from !== undefined) { 36 36 + reqOptsHint.push(`Between ${todayAwareFormat(dayjs(currOpts.from))} and ${todayAwareFormat(dayjs(currOpts.to))}`); 37 37 + } else if(currOpts.to) { 38 38 + reqOptsHint.push(`Until ${todayAwareFormat(dayjs(currOpts.to))}`); 39 39 + } else if(currOpts.to) { 40 40 + reqOptsHint.push(`From ${todayAwareFormat(dayjs(currOpts.from))}`); 41 41 + } 42 42 + 43 43 + if(currOpts.limit !== undefined) { 44 44 + reqOptsHint.push(`Limit ${currOpts.limit}`); 45 45 + } 46 46 + reqLogger.debug(`Fetching => ${reqOptsHint.join(' | ')}`); 47 47 + let results: PagelessTimeRangeListensResult; 48 48 + try { 49 49 + results = await fetcher.getPagelessTimeRangeListens(currOpts); 50 50 + } catch (e) { 51 51 + throw new Error(`API error occurred on Request ${requestCount} with these parameters ${JSON.stringify(currOpts)}`, {cause: e}); 52 52 + } 53 53 + if(initial) { 54 54 + initial = false; 55 55 + const initialFetchLog = []; 56 56 + if(results.meta.total !== undefined) { 57 57 + initialFetchLog.push(`API reported ${results.meta.total} total results`); 58 58 + } 59 59 + if(results.meta.limit !== undefined && results.meta.limit !== currOpts.limit) { 60 60 + initialFetchLog.push(`API reported new limit ${results.meta.limit}`); 61 61 + currOpts.limit = results.meta.limit; 62 62 + } 63 63 + if(initialFetchLog.length > 0) { 64 64 + logger.debug(initialFetchLog.join(' | ')); 65 65 + } 66 66 + } 67 67 + reqLogger.trace(`${results.data.length} results returned${results.data.length === 0 ? ', ending fetch' : ''}`); 68 68 + plays = plays.concat(results.data); 69 69 + if (!results.meta.more) { 70 70 + logger.trace('API indicated no more results, ending fetch'); 71 71 + more = false; 72 72 + } 73 73 + if (results.data.length === 0) { 74 74 + more = false; 75 75 + } 76 76 + // failsafe? 77 77 + if(more && results.meta.limit !== undefined && results.data.length < results.meta.limit) { 78 78 + reqLogger.trace(`Number of returned results was less than reported/defined limit (${results.meta.limit}), ending fetch`); 79 79 + more = false; 80 80 + } 81 81 + 82 82 + if(more && opts.to === undefined && opts.from === undefined) { 83 83 + // only wanted one fetch 84 84 + logger.trace('No to/from defined, ending fetch'); 85 85 + more = false; 86 86 + } 87 87 + 88 88 + if(currOpts.fetchMax !== undefined && plays.length >= currOpts.fetchMax) { 89 89 + logger.trace(`Total fetched (${plays.length}) is >= desired max (${currOpts.fetchMax}), ending fetch`); 90 90 + more = false; 91 91 + } 92 92 + 93 93 + if(more) { 94 94 + if (results.meta.order === undefined || results.meta.order === 'asc') { 95 95 + // if meta.order is ascending then assumption the response returns *oldest first* list 96 96 + // so that the newest play from the response should be used as the new `from` 97 97 + const nextFrom = [...results.data].sort(sortByNewestPlayDate)[0].data.playDate.unix() + 1; 98 98 + currOpts.from = nextFrom; 99 99 + } else { 100 100 + // otherwise, oldest found play should be the new `to` 101 101 + const nextTo = [...results.data].sort(sortByOldestPlayDate)[0].data.playDate.unix() - 1; 102 102 + currOpts.to = nextTo; 103 103 + } 104 104 + } 105 105 + } 106 106 + return plays; 107 107 + } 108 108 + } else if (hasPaginatedTimeRangeListens(fetcher)) { 109 109 + return async (opts: PaginatedListensTimeRangeOptions): Promise<PlayObject[]> => { 110 110 + requestCount = 0; 111 111 + let more = true; 112 112 + let currOpts: PaginatedListensTimeRangeOptions = opts; 113 113 + let initial = true; 114 114 + let timeRangeHint: string; 115 115 + if(currOpts.to !== undefined && currOpts.from !== undefined) { 116 116 + timeRangeHint = `Between ${todayAwareFormat(dayjs.unix(currOpts.from))} and ${todayAwareFormat(dayjs.unix(currOpts.to))}`; 117 117 + } else if(currOpts.to) { 118 118 + timeRangeHint= `Until ${todayAwareFormat(dayjs.unix(currOpts.to))}`; 119 119 + } else if(currOpts.to) { 120 120 + timeRangeHint = `From ${todayAwareFormat(dayjs.unix(currOpts.from))}`; 121 121 + } 122 122 + while (more) { 123 123 + requestCount++; 124 124 + const reqOptsHint: string[] = []; 125 125 + if(currOpts.cursor !== undefined) { 126 126 + `${typeof currOpts.cursor === 'number' ? 'Page' : 'Cursor'} ${currOpts.cursor}`; 127 127 + } 128 128 + if(timeRangeHint !== undefined) { 129 129 + reqOptsHint.push(timeRangeHint); 130 130 + } 131 131 + if(currOpts.limit !== undefined) { 132 132 + reqOptsHint.push(`Limit ${currOpts.limit}`); 133 133 + } 134 134 + reqLogger.debug(`Fetching => ${reqOptsHint.join(' | ')}`); 135 135 + let results: PaginatedTimeRangeListensResult<CursorType>; 136 136 + try { 137 137 + results = await fetcher.getPaginatedTimeRangeListens(currOpts); 138 138 + } catch (e) { 139 139 + throw new Error(`API error occurred on Request ${requestCount} with these parameters ${JSON.stringify(currOpts)}`, {cause: e}); 140 140 + } 141 141 + if(initial) { 142 142 + initial = false; 143 143 + const initialFetchLog = []; 144 144 + if(results.meta.total !== undefined) { 145 145 + initialFetchLog.push(`API reported ${results.meta.total} total results`); 146 146 + } 147 147 + if(results.meta.limit !== undefined && results.meta.limit !== currOpts.limit) { 148 148 + initialFetchLog.push(`API reported new limit ${results.meta.limit}`); 149 149 + currOpts.limit = results.meta.limit; 150 150 + } 151 151 + if(results.meta.cursor !== undefined && results.meta.cursor !== currOpts.cursor) { 152 152 + initialFetchLog.push(`API reported new cursor ${results.meta.cursor}`); 153 153 + currOpts.cursor = results.meta.cursor; 154 154 + } 155 155 + if(initialFetchLog.length > 0) { 156 156 + logger.debug(initialFetchLog.join(' | ')); 157 157 + } 158 158 + } 159 159 + reqLogger.trace(`${results.data.length} results returned${results.data.length === 0 ? ', ending fetch' : ''}`); 160 160 + plays = plays.concat(results.data); 161 161 + if (!results.meta.more) { 162 162 + logger.trace('API indicated no more results, ending fetch'); 163 163 + more = false; 164 164 + } 165 165 + if (results.data.length === 0) { 166 166 + more = false; 167 167 + } 168 168 + // failsafe? 169 169 + if(more && results.meta.limit !== undefined && results.data.length < results.meta.limit) { 170 170 + reqLogger.trace(`Number of returned results was less than reported/defined limit (${results.meta.limit}), ending fetch`); 171 171 + more = false; 172 172 + } 173 173 + 174 174 + if(more && opts.to === undefined && opts.from === undefined) { 175 175 + // only wanted one fetch 176 176 + logger.trace('No to/from defined, ending fetch'); 177 177 + more = false; 178 178 + } 179 179 + 180 180 + if(currOpts.fetchMax !== undefined && plays.length >= currOpts.fetchMax) { 181 181 + logger.trace(`Total fetched (${plays.length}) is >= desired max (${currOpts.fetchMax}), ending fetch`); 182 182 + more = false; 183 183 + } 184 184 + 185 185 + if(more) { 186 186 + if(results.meta.cursorNext !== undefined) { 187 187 + currOpts.cursor = results.meta.cursorNext; 188 188 + } else if(typeof currOpts.cursor === 'number') { 189 189 + currOpts.cursor++; 190 190 + } else { 191 191 + throw new Error('Next cursor is not defined and current cursor is not a number, unable to determine how to increment pagination'); 192 192 + } 193 193 + } 194 194 + } 195 195 + return plays; 196 196 + } 197 197 + } 198 198 + 199 199 + throw new Error('fetcher does not implement pagination interface'); 200 200 + } 201 201 + 202 202 + export interface GroupPlaysTimeRangeOptions { 203 203 + groupDuration?: Duration 204 204 + newPadding?: Duration 205 205 + staleNowBuffer?: number 206 206 + consolidateDuration?: Duration 207 207 + logger?: Logger 208 208 + } 209 209 + 210 210 + export const DEFAULT_GROUP_DURATION = dayjs.duration(15, 'm'); 211 211 + export const DEFAULT_NEW_PADDING = dayjs.duration(10, 'm'); 212 212 + export const DEFAULT_CONSOLIDATE_DURATION = dayjs.duration(3, 'h'); 213 213 + 214 214 + export const groupPlaysToTimeRanges = (plays: PlayObject[], existingRanges: PaginatedTimeRangeOptions[], opts: GroupPlaysTimeRangeOptions = {}) => { 215 215 + const { 216 216 + groupDuration = DEFAULT_GROUP_DURATION, 217 217 + newPadding = DEFAULT_NEW_PADDING, 218 218 + staleNowBuffer = REFRESH_STALE_DEFAULT, 219 219 + consolidateDuration = DEFAULT_CONSOLIDATE_DURATION, 220 220 + logger = loggerNoop 221 221 + } = opts; 222 222 + const newRanges: PaginatedTimeRangeOptions[] = []; 223 223 + 224 224 + const temporallyClosePlaySets: PlayObject[][] = []; 225 225 + 226 226 + const sorted = [...plays]; 227 227 + sorted.sort(sortByOldestPlayDate); 228 228 + 229 229 + for(const p of sorted) { 230 230 + const closePlaySetIndex = temporallyClosePlaySets.findIndex(x => playDateWithinDurationOfAny(p, x, groupDuration)); 231 231 + if(closePlaySetIndex === -1) { 232 232 + temporallyClosePlaySets.push([p]); 233 233 + } else { 234 234 + temporallyClosePlaySets[closePlaySetIndex].push(p); 235 235 + } 236 236 + } 237 237 + 238 238 + // make sure each grouped list is sorted 239 239 + temporallyClosePlaySets.forEach((x) => x.sort(sortByOldestPlayDate)); 240 240 + // sort all lists so oldest list of plays is first 241 241 + temporallyClosePlaySets.sort((a, b) => { 242 242 + const aPlayDate = a[0].data.playDate; 243 243 + const bPlayDate = b[0].data.playDate; 244 244 + if(aPlayDate === undefined && bPlayDate === undefined) { 245 245 + return 0; 246 246 + } 247 247 + if(aPlayDate === undefined) { 248 248 + return 1; 249 249 + } 250 250 + if(bPlayDate === undefined) { 251 251 + return -1; 252 252 + } 253 253 + return aPlayDate.isAfter(bPlayDate) ? 1 : -1 254 254 + }); 255 255 + 256 256 + // try to consolidate lists if they are within a few hours (or consolidateDuration) of their neighbors 257 257 + interface NeighorAcc { 258 258 + lists: PlayObject[][] 259 259 + open: PlayObject[] | undefined 260 260 + } 261 261 + let consolidated: PlayObject[][] = temporallyClosePlaySets; 262 262 + 263 263 + if(consolidated.length > 1) { 264 264 + consolidated = temporallyClosePlaySets.reduce((acc: NeighorAcc, curr, index) => { 265 265 + // if no list is currently being evaluated then open this one and iterate 266 266 + if(index === 0) { 267 267 + acc.open = curr; 268 268 + //return acc; 269 269 + } else { 270 270 + // if a list is open then we need to see if time b/w oldest and newest of curr is less than allowed time 271 271 + 272 272 + if(curr[curr.length - 1].data.playDate.diff(acc.open[0].data.playDate, 's') < consolidateDuration.asSeconds()) { 273 273 + // if less than consolidateDuration then consolidate and iterate 274 274 + acc.open = acc.open.concat(curr); 275 275 + //return acc; 276 276 + } else { 277 277 + // if its not less than consolidateDuration then close list 278 278 + acc.lists.push(acc.open); 279 279 + 280 280 + // and open with curr 281 281 + acc.open = curr; 282 282 + } 283 283 + } 284 284 + 285 285 + if(index === temporallyClosePlaySets.length - 1) { 286 286 + // if this is the last iteration then push current as well 287 287 + acc.lists.push(acc.open) 288 288 + } 289 289 + 290 290 + return acc; 291 291 + 292 292 + 293 293 + }, {lists: [], open: undefined}).lists; 294 294 + logger.trace(`Reduced timerange groups ${temporallyClosePlaySets.length} => ${consolidated.length}`); 295 295 + } 296 296 + 297 297 + for(const tc of consolidated) { 298 298 + let oldest: Dayjs, 299 299 + newest: Dayjs; 300 300 + if(tc.length === 1) { 301 301 + oldest = tc[0].data.playDate; 302 302 + newest = oldest; 303 303 + //newest = tc[0].data.playDate.add(1, 'hour').unix(); 304 304 + } else { 305 305 + oldest = tc[0].data.playDate; 306 306 + newest = tc[tc.length - 1].data.playDate; 307 307 + } 308 308 + 309 309 + let bufferedNewest = newest; 310 310 + if(dayjs().diff(bufferedNewest, 's') < staleNowBuffer) { 311 311 + bufferedNewest = bufferedNewest.subtract(staleNowBuffer, 's'); 312 312 + } 313 313 + const existingWithin = existingRanges.find(x => x.from <= oldest.unix() && x.to >= bufferedNewest.unix()); 314 314 + if(!existingWithin) { 315 315 + newRanges.push({from: oldest.subtract(newPadding).unix(), to: Math.min(newest.add(newPadding).unix(), dayjs().unix())}); 316 316 + } else { 317 317 + newRanges.push(existingWithin); 318 318 + } 319 319 + } 320 320 + 321 321 + return newRanges; 322 322 + }

+5

src/backend/utils/PlayComparisonUtils.ts

Reviewed

··· 6 6 import { compareNormalizedStrings, compareScrobbleArtists, compareScrobbleTracks, compareTracks, normalizeStr, TrackSamenessResults } from "./StringUtils.js"; 7 7 import { ARTIST_WEIGHT, TITLE_WEIGHT } from "../common/infrastructure/Atomic.js"; 8 8 import { StringSamenessResult } from "@foxxmd/string-sameness"; 9 9 + import { Duration } from "dayjs/plugin/duration.js"; 9 10 10 11 11 12 export const metaInvariantTransform = (play: PlayObject): PlayObjectLifecycleless => { ··· 377 378 const albumScore = albumHigh * (albumWeight + albumBonus); 378 379 379 380 return trackScore + artistScore + albumScore; 381 381 + } 382 382 + 383 383 + export const playDateWithinDurationOfAny = (play: PlayObject, plays: PlayObject[], dur: Duration): PlayObject | undefined => { 384 384 + return plays.find(x => Math.abs(x.data.playDate.diff(play.data.playDate, 's')) <= dur.asSeconds()); 380 385 }

+118

src/backend/utils/RequestUtils.ts

Reviewed

··· 1 1 import { Files, File } from "formidable"; 2 2 import VolatileFile from "formidable/VolatileFile.js"; 3 3 import { KNOWN_MEDIA_PROVIDER_URLS } from "../../core/Atomic.js"; 4 4 + import { RequestRetryOptions } from "../common/infrastructure/config/common.js"; 5 5 + import { Logger } from "@foxxmd/logging"; 6 6 + import request, { Request, Response } from 'superagent'; 7 7 + import pRetry, { RetryContext, Options } from 'p-retry'; 8 8 + import { DEFAULT_RETRY_MULTIPLIER } from "../common/infrastructure/Atomic.js"; 9 9 + import { SimpleError } from "../common/errors/MSErrors.js"; 10 10 + import { loggerNoop } from "../common/logging.js"; 11 11 + import { findCauseByFunc } from "./ErrorUtils.js"; 12 12 + import { isSuperAgentResponseError } from "../common/errors/ErrorUtils.js"; 13 13 + import { isNodeNetworkException, NodeNetworkException } from "../common/errors/NodeErrors.js"; 14 14 + import { formatNumber } from "../utils.js"; 15 15 + import { UpstreamError } from "../common/errors/UpstreamError.js"; 4 16 5 17 // typings from Formidable are all nuts. 6 18 // VolatileFile is missing buffer and also does not extend File even though it should ··· 78 90 79 91 export const urlContainsKnownMediaDomain = (url: string | URL): boolean => { 80 92 return urlContainsDomains(url, KNOWN_MEDIA_PROVIDER_URLS); 93 93 + } 94 94 + 95 95 + export interface TryApiCallOptions extends RequestRetryOptions { 96 96 + logger?: Logger 97 97 + logFailure?: boolean | ShouldLogFailure 98 98 + noRetryStatus?: number[] 99 99 + shouldRetry?: ShouldRetryMaybe 100 100 + } 101 101 + 102 102 + export const NO_RETRY_HTTP_STATUS = [400,403,401]; 103 103 + 104 104 + export type ShouldLogFailure = (context: RetryContext) => boolean; 105 105 + export type ShouldRetryMaybe = (context: RetryContext) => boolean | undefined; 106 106 + 107 107 + export const tryApiCall = async <T = Response>(reqFunc: () => T, opts: TryApiCallOptions = {}): Promise<T> => { 108 108 + const { 109 109 + maxRequestRetries: retries = 2, 110 110 + retryMultiplier = DEFAULT_RETRY_MULTIPLIER, 111 111 + logFailure = true, 112 112 + logger = loggerNoop, 113 113 + noRetryStatus = NO_RETRY_HTTP_STATUS, 114 114 + shouldRetry 115 115 + } = opts; 116 116 + 117 117 + const retryOpts: Options = { 118 118 + retries: retries, 119 119 + factor: retryMultiplier, 120 120 + minTimeout: 1000, 121 121 + maxRetryTime: 30000, 122 122 + } 123 123 + 124 124 + const getDelay = curriedDelay(retryOpts); 125 125 + 126 126 + try { 127 127 + return await pRetry(() => reqFunc(), { 128 128 + ...retryOpts, 129 129 + shouldRetry(context) { 130 130 + let willRetry: boolean; 131 131 + if (shouldRetry !== undefined) { 132 132 + try { 133 133 + const res = shouldRetry(context); 134 134 + if (typeof res === 'boolean') { 135 135 + willRetry = res; 136 136 + } 137 137 + } catch (e) { 138 138 + logger.warn(new SimpleError('Failed to evaluate shouldRetry function. Falling back to default retry logic.', { cause: e })); 139 139 + } 140 140 + } 141 141 + if(willRetry === undefined) { 142 142 + const cause = findCauseByFunc<request.ResponseError | NodeNetworkException>(context.error, (e) => isSuperAgentResponseError(e) || isNodeNetworkException(e)); 143 143 + if (cause === undefined) { 144 144 + willRetry = false; 145 145 + } else if (isNodeNetworkException(cause)) { 146 146 + willRetry = true; 147 147 + } else if (noRetryStatus.includes(cause.status)) { 148 148 + willRetry = false; 149 149 + } else { 150 150 + willRetry = true; 151 151 + } 152 152 + } 153 153 + 154 154 + if(willRetry) { 155 155 + let shouldLog: boolean = false; 156 156 + if (logFailure === true) { 157 157 + shouldLog = true; 158 158 + } 159 159 + else if (typeof logFailure === 'function') { 160 160 + try { 161 161 + shouldLog = logFailure(context); 162 162 + } catch (e) { 163 163 + logger.warn(new SimpleError('Failed to evaluate logFailure function (lol). Falling back to logging original error for context.', { cause: e })); 164 164 + shouldLog = true; 165 165 + } 166 166 + } 167 167 + if (shouldLog) { 168 168 + logger.warn(new SimpleError(`Request attempt ${context.attemptNumber} failed. ${context.retriesLeft} retries left. Waiting ${getDelay(context.retriesConsumed + 1)}s before next try.`, { cause: context.error, shortStack: true })); 169 169 + } 170 170 + } 171 171 + 172 172 + return willRetry; 173 173 + } 174 174 + }) 175 175 + } catch (e) { 176 176 + throw e; 177 177 + } 178 178 + } 179 179 + 180 180 + const calculateDelay = (retriesConsumed: number, options: Options = {}) => { 181 181 + const attempt = Math.max(1, retriesConsumed + 1); 182 182 + const random = (options.randomize ?? false) ? (Math.random() + 1) : 1; 183 183 + 184 184 + let timeout = Math.round(random * (options.minTimeout ?? 1000) * ((options.factor ?? 2) ** (attempt - 1))); 185 185 + timeout = Math.min(timeout, options.maxTimeout ?? Number.POSITIVE_INFINITY); 186 186 + 187 187 + return timeout; 188 188 + } 189 189 + 190 190 + const curriedDelay = (options: Options = {}) => { 191 191 + return (retriesConsumed: number) => formatNumber(calculateDelay(retriesConsumed, options) / 1000); 192 192 + } 193 193 + 194 194 + export const noRetryOnUpstreamError = (context: RetryContext): boolean | undefined => { 195 195 + if(context.error instanceof UpstreamError && context.error.showStopper === true) { 196 196 + return true; 197 197 + } 198 198 + return undefined; 81 199 }

+6

src/backend/utils/TimeUtils.ts

Reviewed

··· 15 15 TA_NONE, 16 16 TemporalAccuracy, 17 17 TemporalPlayComparison, 18 18 + UnixTimestamp, 18 19 } from "../../core/Atomic.js"; 19 20 import { capitalize } from "../../core/StringUtils.js"; 20 21 import { ··· 412 413 } 413 414 414 415 return [absPlayed || majorityDurationPercent, `${hintStart}${hints.join(' and ')}`]; 416 416 + } 417 417 + 418 418 + /** Convert unix timestamp in microseconds to unix timestamp in seconds */ 419 419 + export const usecToUnix = (usec: number): UnixTimestamp => { 420 420 + return Math.floor(usec / 1000); 415 421 }

+3 -2

src/client/logs/LogsSection.tsx

Reviewed

··· 46 46 logs, 47 47 settings, 48 48 settings: { 49 49 - limit = 50 49 49 + limit = 50, 50 50 + level 50 51 } 51 52 } = props; 52 53 53 54 const [logList, setLogList] = useState(logBuffer); 54 54 - const [logLevel, setLogLevel] = useState('debug') 55 55 + const [logLevel, setLogLevel] = useState(level ?? 'trace') 55 56 56 57 useGetLogsQuery(undefined); 57 58

+1 -1

src/client/logs/logDucks.ts

Reviewed

··· 9 9 data: (LogDataPretty & {levelLabel: string})[], 10 10 settings: LogOutputConfig 11 11 } 12 12 - const initialState: LogsState = {data: [], settings: {level: 'debug', sort: 'asc', limit: 50}}; 12 12 + const initialState: LogsState = {data: [], settings: {level: 'trace', sort: 'desc', limit: 50}}; 13 13 const logsReducer = createReducer(initialState, (builder) => { 14 14 builder 15 15 .addMatcher(

+8 -1

src/core/Atomic.ts

Reviewed

··· 195 195 196 196 export interface PlayMeta { 197 197 source?: string 198 198 + sourceSOT?: SOURCE_SOT_TYPES 198 199 199 200 /* 200 201 * If applicable, the name of the Service providing the track (Spotify, Tidal, etc...) ··· 327 328 reason?: string 328 329 closestMatchedPlay?: PlayObjectLifecycleless 329 330 transformedPlay?: PlayObjectLifecycleless 331 331 + summary?: String 330 332 } 331 333 332 334 export type ScrobbleTsSOC = 1 | 2; ··· 562 564 'play.google.com', 563 565 'listenbrainz.org', 564 566 'musicbrainz.org' 565 565 - ]; 567 567 + ]; 568 568 + 569 569 + /** Number of SECONDS since 1970 */ 570 570 + export type UnixTimestamp = number; 571 571 + 572 572 + export type Writeable<T> = { -readonly [P in keyof T]: T[P] };