soundcloud.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710
  1. import mongoose from "mongoose";
  2. import async from "async";
  3. import config from "config";
  4. import sckey from "soundcloud-key-fetch";
  5. import * as rax from "retry-axios";
  6. import axios from "axios";
  7. import CoreClass from "../core";
  8. let SoundCloudModule;
  9. let DBModule;
  10. let CacheModule;
  11. let MediaModule;
  12. const soundcloudTrackObjectToMusareTrackObject = soundcloudTrackObject => {
  13. const {
  14. id,
  15. title,
  16. artwork_url: artworkUrl,
  17. created_at: createdAt,
  18. duration,
  19. genre,
  20. kind,
  21. license,
  22. likes_count: likesCount,
  23. playback_count: playbackCount,
  24. public: _public,
  25. tag_list: tagList,
  26. user_id: userId,
  27. user,
  28. track_format: trackFormat,
  29. permalink,
  30. monetization_model: monetizationModel,
  31. policy,
  32. streamable,
  33. sharing,
  34. state,
  35. embeddable_by: embeddableBy
  36. } = soundcloudTrackObject;
  37. return {
  38. trackId: id,
  39. title,
  40. artworkUrl,
  41. soundcloudCreatedAt: new Date(createdAt),
  42. duration: duration / 1000,
  43. genre,
  44. kind,
  45. license,
  46. likesCount,
  47. playbackCount,
  48. public: _public,
  49. tagList,
  50. userId,
  51. username: user.username,
  52. userPermalink: user.permalink,
  53. trackFormat,
  54. permalink,
  55. monetizationModel,
  56. policy,
  57. streamable,
  58. sharing,
  59. state,
  60. embeddableBy
  61. };
  62. };
  63. class RateLimitter {
  64. /**
  65. * Constructor
  66. *
  67. * @param {number} timeBetween - The time between each allowed YouTube request
  68. */
  69. constructor(timeBetween) {
  70. this.dateStarted = Date.now();
  71. this.timeBetween = timeBetween;
  72. }
  73. /**
  74. * Returns a promise that resolves whenever the ratelimit of a YouTube request is done
  75. *
  76. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  77. */
  78. continue() {
  79. return new Promise(resolve => {
  80. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  81. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  82. });
  83. }
  84. /**
  85. * Restart the rate limit timer
  86. */
  87. restart() {
  88. this.dateStarted = Date.now();
  89. }
  90. }
  91. class _SoundCloudModule extends CoreClass {
  92. // eslint-disable-next-line require-jsdoc
  93. constructor() {
  94. super("soundcloud");
  95. SoundCloudModule = this;
  96. }
  97. /**
  98. * Initialises the soundcloud module
  99. *
  100. * @returns {Promise} - returns promise (reject, resolve)
  101. */
  102. async initialize() {
  103. DBModule = this.moduleManager.modules.db;
  104. CacheModule = this.moduleManager.modules.cache;
  105. MediaModule = this.moduleManager.modules.media;
  106. this.soundcloudTrackModel = this.SoundCloudTrackModel = await DBModule.runJob("GET_MODEL", {
  107. modelName: "soundcloudTrack"
  108. });
  109. return new Promise((resolve, reject) => {
  110. this.rateLimiter = new RateLimitter(config.get("apis.soundcloud.rateLimit"));
  111. this.requestTimeout = config.get("apis.soundcloud.requestTimeout");
  112. this.axios = axios.create();
  113. this.axios.defaults.raxConfig = {
  114. instance: this.axios,
  115. retry: config.get("apis.soundcloud.retryAmount"),
  116. noResponseRetries: config.get("apis.soundcloud.retryAmount")
  117. };
  118. rax.attach(this.axios);
  119. this.apiKey = null;
  120. SoundCloudModule.runJob("TEST_SOUNDCLOUD_API_KEY", {}, null, -1)
  121. .then(result => {
  122. if (result) {
  123. resolve();
  124. return;
  125. }
  126. SoundCloudModule.runJob("GENERATE_SOUNDCLOUD_API_KEY", {}, null, -1)
  127. .then(() => {
  128. resolve();
  129. })
  130. .catch(reject);
  131. })
  132. .catch(reject);
  133. });
  134. }
  135. /**
  136. * Generates/fetches a new SoundCloud API key
  137. *
  138. * @returns {Promise} - returns promise (reject, resolve)
  139. */
  140. GENERATE_SOUNDCLOUD_API_KEY() {
  141. return new Promise((resolve, reject) => {
  142. this.log("INFO", "Fetching new SoundCloud API key.");
  143. sckey
  144. .fetchKey()
  145. .then(soundcloudApiKey => {
  146. if (!soundcloudApiKey) {
  147. this.log("ERROR", "Couldn't fetch new SoundCloud API key.");
  148. reject(new Error("Couldn't fetch SoundCloud key."));
  149. return;
  150. }
  151. SoundCloudModule.soundcloudApiKey = soundcloudApiKey;
  152. CacheModule.runJob("SET", { key: "soundcloudApiKey", value: soundcloudApiKey }, this)
  153. .then(() => {
  154. SoundCloudModule.runJob("TEST_SOUNDCLOUD_API_KEY", {}, this).then(result => {
  155. if (!result) {
  156. this.log("ERROR", "Fetched SoundCloud API key is invalid.");
  157. reject(new Error("SoundCloud key isn't valid."));
  158. } else {
  159. this.log("INFO", "Fetched new valid SoundCloud API key.");
  160. resolve();
  161. }
  162. });
  163. })
  164. .catch(err => {
  165. this.log("ERROR", `Couldn't set new SoundCloud API key in cache. Error: ${err.message}`);
  166. reject(err);
  167. });
  168. })
  169. .catch(err => {
  170. this.log("ERROR", `Couldn't fetch new SoundCloud API key. Error: ${err.message}`);
  171. reject(new Error("Couldn't fetch SoundCloud key."));
  172. });
  173. });
  174. }
  175. /**
  176. * Tests the stored SoundCloud API key
  177. *
  178. * @returns {Promise} - returns promise (reject, resolve)
  179. */
  180. TEST_SOUNDCLOUD_API_KEY() {
  181. return new Promise((resolve, reject) => {
  182. this.log("INFO", "Testing SoundCloud API key.");
  183. CacheModule.runJob("GET", { key: "soundcloudApiKey" }, this).then(soundcloudApiKey => {
  184. if (!soundcloudApiKey) {
  185. this.log("ERROR", "No SoundCloud API key found in cache.");
  186. resolve(false);
  187. return;
  188. }
  189. SoundCloudModule.soundcloudApiKey = soundcloudApiKey;
  190. sckey
  191. .testKey(soundcloudApiKey)
  192. .then(res => {
  193. this.log("INFO", `Tested SoundCloud API key. Result: ${res}`);
  194. resolve(res);
  195. })
  196. .catch(err => {
  197. this.log("ERROR", `Testing SoundCloud API key error: ${err.message}`);
  198. reject(err);
  199. });
  200. });
  201. });
  202. }
  203. /**
  204. * Perform SoundCloud API get track request
  205. *
  206. * @param {object} payload - object that contains the payload
  207. * @param {string} payload.trackId - the SoundCloud track id to get
  208. * @returns {Promise} - returns promise (reject, resolve)
  209. */
  210. API_GET_TRACK(payload) {
  211. return new Promise((resolve, reject) => {
  212. const { trackId } = payload;
  213. SoundCloudModule.runJob(
  214. "API_CALL",
  215. {
  216. url: `https://api-v2.soundcloud.com/tracks/${trackId}`
  217. },
  218. this
  219. )
  220. .then(response => {
  221. resolve(response);
  222. })
  223. .catch(err => {
  224. reject(err);
  225. });
  226. });
  227. }
  228. /**
  229. * Perform SoundCloud API call
  230. *
  231. * @param {object} payload - object that contains the payload
  232. * @param {string} payload.url - request url
  233. * @param {object} payload.params - request parameters
  234. * @returns {Promise} - returns promise (reject, resolve)
  235. */
  236. API_CALL(payload) {
  237. return new Promise((resolve, reject) => {
  238. const { url } = payload;
  239. const { soundcloudApiKey } = SoundCloudModule;
  240. const params = {
  241. client_id: soundcloudApiKey
  242. };
  243. SoundCloudModule.axios
  244. .get(url, {
  245. params,
  246. timeout: SoundCloudModule.requestTimeout
  247. })
  248. .then(response => {
  249. if (response.data.error) {
  250. reject(new Error(response.data.error));
  251. } else {
  252. resolve({ response });
  253. }
  254. })
  255. .catch(err => {
  256. reject(err);
  257. });
  258. // }
  259. });
  260. }
  261. /**
  262. * Create SoundCloud track
  263. *
  264. * @param {object} payload - an object containing the payload
  265. * @param {object} payload.soundcloudTrack - the soundcloudTrack object
  266. * @returns {Promise} - returns a promise (resolve, reject)
  267. */
  268. CREATE_TRACK(payload) {
  269. return new Promise((resolve, reject) => {
  270. async.waterfall(
  271. [
  272. next => {
  273. const { soundcloudTrack } = payload;
  274. if (typeof soundcloudTrack !== "object") next("Invalid soundcloudTrack type");
  275. else {
  276. SoundCloudModule.soundcloudTrackModel.insertMany(soundcloudTrack, next);
  277. }
  278. },
  279. (soundcloudTracks, next) => {
  280. const mediaSources = soundcloudTracks.map(
  281. soundcloudTrack => `soundcloud:${soundcloudTrack.trackId}`
  282. );
  283. async.eachLimit(
  284. mediaSources,
  285. 2,
  286. (mediaSource, next) => {
  287. MediaModule.runJob("RECALCULATE_RATINGS", { mediaSource }, this)
  288. .then(() => next())
  289. .catch(next);
  290. },
  291. err => {
  292. if (err) next(err);
  293. else next(null, soundcloudTracks);
  294. }
  295. );
  296. }
  297. ],
  298. (err, soundcloudTracks) => {
  299. if (err) reject(new Error(err));
  300. else resolve({ soundcloudTracks });
  301. }
  302. );
  303. });
  304. }
  305. /**
  306. * Get SoundCloud track
  307. *
  308. * @param {object} payload - an object containing the payload
  309. * @param {string} payload.identifier - the soundcloud track ObjectId or track id
  310. * @param {boolean} payload.createMissing - attempt to fetch and create track if not in db
  311. * @returns {Promise} - returns a promise (resolve, reject)
  312. */
  313. GET_TRACK(payload) {
  314. return new Promise((resolve, reject) => {
  315. async.waterfall(
  316. [
  317. next => {
  318. const query = mongoose.isObjectIdOrHexString(payload.identifier)
  319. ? { _id: payload.identifier }
  320. : { trackId: payload.identifier };
  321. return SoundCloudModule.soundcloudTrackModel.findOne(query, next);
  322. },
  323. (track, next) => {
  324. if (track) return next(null, track, false);
  325. if (mongoose.isObjectIdOrHexString(payload.identifier) || !payload.createMissing)
  326. return next("SoundCloud track not found.");
  327. return SoundCloudModule.runJob("API_GET_TRACK", { trackId: payload.identifier }, this)
  328. .then(({ response }) => {
  329. const { data } = response;
  330. if (!data || !data.id)
  331. return next("The specified track does not exist or cannot be publicly accessed.");
  332. const soundcloudTrack = soundcloudTrackObjectToMusareTrackObject(data);
  333. return next(null, false, soundcloudTrack);
  334. })
  335. .catch(next);
  336. },
  337. (track, soundcloudTrack, next) => {
  338. if (track) return next(null, track, true);
  339. return SoundCloudModule.runJob("CREATE_TRACK", { soundcloudTrack }, this)
  340. .then(res => {
  341. if (res.soundcloudTracks.length === 1) next(null, res.soundcloudTracks[0], false);
  342. else next("SoundCloud track not found.");
  343. })
  344. .catch(next);
  345. }
  346. ],
  347. (err, track, existing) => {
  348. if (err) reject(new Error(err));
  349. else if (track.policy === "SNIP") reject(new Error("Track is premium-only."));
  350. else resolve({ track, existing });
  351. }
  352. );
  353. });
  354. }
  355. /**
  356. * Tries to get a SoundCloud track from a URL
  357. *
  358. * @param {object} payload - object that contains the payload
  359. * @param {string} payload.identifier - the SoundCloud track URL
  360. * @returns {Promise} - returns promise (reject, resolve)
  361. */
  362. GET_TRACK_FROM_URL(payload) {
  363. return new Promise((resolve, reject) => {
  364. const scRegex =
  365. /soundcloud\.com\/(?<userPermalink>[A-Za-z0-9]+([-_][A-Za-z0-9]+)*)\/(?<permalink>[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*)/;
  366. async.waterfall(
  367. [
  368. next => {
  369. const match = scRegex.exec(payload.identifier);
  370. if (!match || !match.groups) {
  371. next("Invalid SoundCloud URL.");
  372. return;
  373. }
  374. const { userPermalink, permalink } = match.groups;
  375. SoundCloudModule.soundcloudTrackModel.findOne({ userPermalink, permalink }, next);
  376. },
  377. (_dbTrack, next) => {
  378. if (_dbTrack) {
  379. next(null, _dbTrack, true);
  380. return;
  381. }
  382. SoundCloudModule.runJob("API_RESOLVE", { url: payload.identifier }, this)
  383. .then(({ response }) => {
  384. const { data } = response;
  385. if (!data || !data.id) {
  386. next("The provided URL does not exist or cannot be accessed.");
  387. return;
  388. }
  389. if (data.kind !== "track") {
  390. next(`Invalid URL provided. Kind got: ${data.kind}.`);
  391. return;
  392. }
  393. // TODO get more data here
  394. const { id: trackId } = data;
  395. SoundCloudModule.soundcloudTrackModel.findOne({ trackId }, (err, dbTrack) => {
  396. if (err) next(err);
  397. else if (dbTrack) {
  398. next(null, dbTrack, true);
  399. } else {
  400. const soundcloudTrack = soundcloudTrackObjectToMusareTrackObject(data);
  401. SoundCloudModule.runJob("CREATE_TRACK", { soundcloudTrack }, this)
  402. .then(res => {
  403. if (res.soundcloudTracks.length === 1)
  404. next(null, res.soundcloudTracks[0], false);
  405. else next("SoundCloud track not found.");
  406. })
  407. .catch(next);
  408. }
  409. });
  410. })
  411. .catch(next);
  412. }
  413. ],
  414. (err, track, existing) => {
  415. if (err) reject(new Error(err));
  416. else if (track.policy === "SNIP") reject(new Error("Track is premium-only."));
  417. else resolve({ track, existing });
  418. }
  419. );
  420. });
  421. }
  422. /**
  423. * Returns an array of songs taken from a SoundCloud playlist
  424. *
  425. * @param {object} payload - object that contains the payload
  426. * @param {string} payload.url - the url of the SoundCloud playlist
  427. * @returns {Promise} - returns promise (reject, resolve)
  428. */
  429. GET_PLAYLIST(payload) {
  430. return new Promise((resolve, reject) => {
  431. async.waterfall(
  432. [
  433. next => {
  434. SoundCloudModule.runJob("API_RESOLVE", { url: payload.url }, this)
  435. .then(async ({ response }) => {
  436. const { data } = response;
  437. if (!data || !data.id)
  438. return next("The provided URL does not exist or cannot be accessed.");
  439. let tracks;
  440. if (data.kind === "user")
  441. tracks = (
  442. await SoundCloudModule.runJob(
  443. "GET_ARTIST_TRACKS",
  444. {
  445. artistId: data.id
  446. },
  447. this
  448. )
  449. ).tracks;
  450. else if (data.kind !== "playlist" && data.kind !== "system-playlist")
  451. return next(`Invalid URL provided. Kind got: ${data.kind}.`);
  452. else tracks = data.tracks;
  453. const soundcloudTrackIds = tracks.map(track => track.id);
  454. return next(null, soundcloudTrackIds);
  455. })
  456. .catch(next);
  457. }
  458. ],
  459. (err, soundcloudTrackIds) => {
  460. if (err && err !== true) {
  461. SoundCloudModule.log(
  462. "ERROR",
  463. "GET_PLAYLIST",
  464. "Some error has occurred.",
  465. typeof err === "string" ? err : err.message
  466. );
  467. reject(new Error(typeof err === "string" ? err : err.message));
  468. } else {
  469. resolve({ songs: soundcloudTrackIds });
  470. }
  471. }
  472. );
  473. // kind;
  474. });
  475. }
  476. /**
  477. * Returns an array of songs taken from a SoundCloud artist
  478. *
  479. * @param {object} payload - object that contains the payload
  480. * @param {string} payload.artistId - the id of the SoundCloud artist
  481. * @returns {Promise} - returns promise (reject, resolve)
  482. */
  483. GET_ARTIST_TRACKS(payload) {
  484. return new Promise((resolve, reject) => {
  485. async.waterfall(
  486. [
  487. next => {
  488. let first = true;
  489. let nextHref = null;
  490. let tracks = [];
  491. async.whilst(
  492. next => {
  493. if (nextHref || first) next(null, true);
  494. else next(null, false);
  495. },
  496. next => {
  497. let job;
  498. if (first) {
  499. job = SoundCloudModule.runJob(
  500. "API_GET_ARTIST_TRACKS",
  501. { artistId: payload.artistId },
  502. this
  503. );
  504. first = false;
  505. } else job = SoundCloudModule.runJob("API_GET_ARTIST_TRACKS", { nextHref }, this);
  506. job.then(({ response }) => {
  507. const { data } = response;
  508. const { collection, next_href: _nextHref } = data;
  509. nextHref = _nextHref;
  510. tracks = tracks.concat(collection);
  511. setTimeout(() => {
  512. next();
  513. }, 500);
  514. }).catch(err => {
  515. next(err);
  516. });
  517. },
  518. err => {
  519. if (err) return next(err);
  520. return next(null, tracks);
  521. }
  522. );
  523. }
  524. ],
  525. (err, tracks) => {
  526. if (err && err !== true) {
  527. SoundCloudModule.log(
  528. "ERROR",
  529. "GET_ARTIST_TRACKS",
  530. "Some error has occurred.",
  531. typeof err === "string" ? err : err.message
  532. );
  533. reject(new Error(typeof err === "string" ? err : err.message));
  534. } else {
  535. resolve({ tracks });
  536. }
  537. }
  538. );
  539. });
  540. }
  541. /**
  542. * Get Soundcloud artists
  543. *
  544. * @param {object} payload - an object containing the payload
  545. * @param {Array} payload.userPermalinks - an array of Soundcloud user permalinks
  546. * @returns {Promise} - returns a promise (resolve, reject)
  547. */
  548. async GET_ARTISTS_FROM_PERMALINKS(payload) {
  549. const getArtists = async userPermalinks => {
  550. const jobsToRun = [];
  551. userPermalinks.forEach(userPermalink => {
  552. const url = `https://soundcloud.com/${userPermalink}`;
  553. jobsToRun.push(SoundCloudModule.runJob("API_RESOLVE", { url }, this));
  554. });
  555. const jobResponses = await Promise.all(jobsToRun);
  556. return jobResponses
  557. .map(jobResponse => jobResponse.response.data)
  558. .map(artist => ({
  559. artistId: artist.id,
  560. username: artist.username,
  561. avatarUrl: artist.avatar_url,
  562. permalink: artist.permalink,
  563. rawData: artist
  564. }));
  565. };
  566. const { userPermalinks } = payload;
  567. const existingArtists = [];
  568. const existingUserPermalinks = existingArtists.map(existingArtists => existingArtists.userPermalink);
  569. // const existingArtistsObjectIds = existingArtists.map(existingArtists => existingArtists._id.toString());
  570. if (userPermalinks.length === existingArtists.length) return { artists: existingArtists };
  571. const missingUserPermalinks = userPermalinks.filter(
  572. userPermalink => existingUserPermalinks.indexOf(userPermalink) === -1
  573. );
  574. if (missingUserPermalinks.length === 0) return { videos: existingArtists };
  575. const newArtists = await getArtists(missingUserPermalinks);
  576. // await SoundcloudModule.soundcloudArtistsModel.insertMany(newArtists);
  577. return { artists: existingArtists.concat(newArtists) };
  578. }
  579. /**
  580. * @param {object} payload - object that contains the payload
  581. * @param {string} payload.url - the url of the SoundCloud resource
  582. * @returns {Promise} - returns a promise (resolve, reject)
  583. */
  584. API_RESOLVE(payload) {
  585. return new Promise((resolve, reject) => {
  586. const { url } = payload;
  587. SoundCloudModule.runJob(
  588. "API_CALL",
  589. {
  590. url: `https://api-v2.soundcloud.com/resolve?url=${encodeURIComponent(url)}`
  591. },
  592. this
  593. )
  594. .then(response => {
  595. resolve(response);
  596. })
  597. .catch(err => {
  598. reject(err);
  599. });
  600. });
  601. }
  602. /**
  603. * Calls the API_CALL with the proper URL to get artist/user tracks
  604. *
  605. * @param {object} payload - object that contains the payload
  606. * @param {string} payload.artistId - the id of the SoundCloud artist
  607. * @param {string} payload.nextHref - the next url to call
  608. * @returns {Promise} - returns a promise (resolve, reject)
  609. */
  610. API_GET_ARTIST_TRACKS(payload) {
  611. return new Promise((resolve, reject) => {
  612. const { artistId, nextHref } = payload;
  613. SoundCloudModule.runJob(
  614. "API_CALL",
  615. {
  616. url: artistId
  617. ? `https://api-v2.soundcloud.com/users/${artistId}/tracks?access=playable&limit=50`
  618. : nextHref
  619. },
  620. this
  621. )
  622. .then(response => {
  623. resolve(response);
  624. })
  625. .catch(err => {
  626. reject(err);
  627. });
  628. });
  629. }
  630. }
  631. export default new _SoundCloudModule();