youtube.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434
  1. import async from "async";
  2. import config from "config";
  3. import * as rax from "retry-axios";
  4. import axios from "axios";
  5. import CoreClass from "../core";
  6. class RateLimitter {
  7. /**
  8. * Constructor
  9. *
  10. * @param {number} timeBetween - The time between each allowed YouTube request
  11. */
  12. constructor(timeBetween) {
  13. this.dateStarted = Date.now();
  14. this.timeBetween = timeBetween;
  15. }
  16. /**
  17. * Returns a promise that resolves whenever the ratelimit of a YouTube request is done
  18. *
  19. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  20. */
  21. continue() {
  22. return new Promise(resolve => {
  23. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  24. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  25. });
  26. }
  27. /**
  28. * Restart the rate limit timer
  29. */
  30. restart() {
  31. this.dateStarted = Date.now();
  32. }
  33. }
  34. let YouTubeModule;
  35. class _YouTubeModule extends CoreClass {
  36. // eslint-disable-next-line require-jsdoc
  37. constructor() {
  38. super("youtube", {
  39. concurrency: 1,
  40. priorities: {
  41. GET_PLAYLIST: 11
  42. }
  43. });
  44. YouTubeModule = this;
  45. }
  46. /**
  47. * Initialises the activities module
  48. *
  49. * @returns {Promise} - returns promise (reject, resolve)
  50. */
  51. initialize() {
  52. return new Promise(resolve => {
  53. this.rateLimiter = new RateLimitter(config.get("apis.youtube.rateLimit"));
  54. this.requestTimeout = config.get("apis.youtube.requestTimeout");
  55. this.axios = axios.create();
  56. this.axios.defaults.raxConfig = {
  57. instance: this.axios,
  58. retry: config.get("apis.youtube.retryAmount"),
  59. noResponseRetries: config.get("apis.youtube.retryAmount")
  60. };
  61. rax.attach(this.axios);
  62. resolve();
  63. });
  64. }
  65. /**
  66. * Fetches a list of songs from Youtube's API
  67. *
  68. * @param {object} payload - object that contains the payload
  69. * @param {string} payload.query - the query we'll pass to youtubes api
  70. * @param {string} payload.pageToken - (optional) if this exists, will search search youtube for a specific page reference
  71. * @returns {Promise} - returns promise (reject, resolve)
  72. */
  73. SEARCH(payload) {
  74. const params = {
  75. part: "snippet",
  76. q: payload.query,
  77. key: config.get("apis.youtube.key"),
  78. type: "video",
  79. maxResults: 10
  80. };
  81. if (payload.pageToken) params.pageToken = payload.pageToken;
  82. return new Promise((resolve, reject) =>
  83. YouTubeModule.rateLimiter.continue().then(() => {
  84. YouTubeModule.rateLimiter.restart();
  85. YouTubeModule.axios
  86. .get("https://www.googleapis.com/youtube/v3/search", {
  87. params,
  88. raxConfig: {
  89. onRetryAttempt: err => {
  90. const cfg = rax.getConfig(err);
  91. YouTubeModule.log(
  92. "ERROR",
  93. "SEARCH",
  94. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  95. );
  96. }
  97. }
  98. })
  99. .then(res => {
  100. if (res.data.err) {
  101. YouTubeModule.log("ERROR", "SEARCH", `${res.data.error.message}`);
  102. return reject(new Error("An error has occured. Please try again later."));
  103. }
  104. return resolve(res.data);
  105. })
  106. .catch(err => {
  107. YouTubeModule.log("ERROR", "SEARCH", `${err.message}`);
  108. return reject(new Error("An error has occured. Please try again later."));
  109. });
  110. })
  111. );
  112. }
  113. /**
  114. * Gets the details of a song using the YouTube API
  115. *
  116. * @param {object} payload - object that contains the payload
  117. * @param {string} payload.youtubeId - the YouTube API id of the song
  118. * @returns {Promise} - returns promise (reject, resolve)
  119. */
  120. GET_SONG(payload) {
  121. return new Promise((resolve, reject) => {
  122. const params = {
  123. part: "snippet,contentDetails,statistics,status",
  124. id: payload.youtubeId,
  125. key: config.get("apis.youtube.key")
  126. };
  127. if (payload.pageToken) params.pageToken = payload.pageToken;
  128. YouTubeModule.rateLimiter.continue().then(() => {
  129. YouTubeModule.rateLimiter.restart();
  130. YouTubeModule.axios
  131. .get("https://www.googleapis.com/youtube/v3/videos", {
  132. params,
  133. timeout: YouTubeModule.requestTimeout,
  134. raxConfig: {
  135. onRetryAttempt: err => {
  136. const cfg = rax.getConfig(err);
  137. YouTubeModule.log(
  138. "ERROR",
  139. "GET_SONG",
  140. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  141. );
  142. }
  143. }
  144. })
  145. .then(res => {
  146. if (res.data.error) {
  147. YouTubeModule.log("ERROR", "GET_SONG", `${res.data.error.message}`);
  148. return reject(new Error("An error has occured. Please try again later."));
  149. }
  150. if (res.data.items[0] === undefined)
  151. return reject(
  152. new Error("The specified video does not exist or cannot be publicly accessed.")
  153. );
  154. // TODO Clean up duration converter
  155. let dur = res.data.items[0].contentDetails.duration;
  156. dur = dur.replace("PT", "");
  157. let duration = 0;
  158. dur = dur.replace(/([\d]*)H/, (v, v2) => {
  159. v2 = Number(v2);
  160. duration = v2 * 60 * 60;
  161. return "";
  162. });
  163. dur = dur.replace(/([\d]*)M/, (v, v2) => {
  164. v2 = Number(v2);
  165. duration += v2 * 60;
  166. return "";
  167. });
  168. // eslint-disable-next-line no-unused-vars
  169. dur = dur.replace(/([\d]*)S/, (v, v2) => {
  170. v2 = Number(v2);
  171. duration += v2;
  172. return "";
  173. });
  174. const song = {
  175. youtubeId: res.data.items[0].id,
  176. title: res.data.items[0].snippet.title,
  177. thumbnail: res.data.items[0].snippet.thumbnails.default.url,
  178. duration
  179. };
  180. return resolve({ song });
  181. })
  182. .catch(err => {
  183. // console.log(111, err, payload);
  184. YouTubeModule.log("ERROR", "GET_SONG", `${err.message}`);
  185. return reject(new Error("An error has occured. Please try again later."));
  186. });
  187. });
  188. });
  189. }
  190. /**
  191. * Returns an array of songs taken from a YouTube playlist
  192. *
  193. * @param {object} payload - object that contains the payload
  194. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the playlist
  195. * @param {string} payload.url - the url of the YouTube playlist
  196. * @returns {Promise} - returns promise (reject, resolve)
  197. */
  198. GET_PLAYLIST(payload) {
  199. return new Promise((resolve, reject) => {
  200. const regex = new RegExp(`[\\?&]list=([^&#]*)`);
  201. const splitQuery = regex.exec(payload.url);
  202. if (!splitQuery) {
  203. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Invalid YouTube playlist URL query.");
  204. return reject(new Error("Invalid playlist URL."));
  205. }
  206. const playlistId = splitQuery[1];
  207. return async.waterfall(
  208. [
  209. next => {
  210. let songs = [];
  211. let nextPageToken = "";
  212. async.whilst(
  213. next => {
  214. YouTubeModule.log(
  215. "INFO",
  216. `Getting playlist progress for job (${this.toString()}): ${
  217. songs.length
  218. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  219. );
  220. next(null, nextPageToken !== undefined);
  221. },
  222. next => {
  223. // Add 250ms delay between each job request
  224. setTimeout(() => {
  225. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  226. .then(response => {
  227. songs = songs.concat(response.songs);
  228. nextPageToken = response.nextPageToken;
  229. next();
  230. })
  231. .catch(err => next(err));
  232. }, 250);
  233. },
  234. err => next(err, songs)
  235. );
  236. },
  237. (songs, next) =>
  238. next(
  239. null,
  240. songs.map(song => song.contentDetails.videoId)
  241. ),
  242. (songs, next) => {
  243. if (!payload.musicOnly) return next(true, { songs });
  244. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  245. .then(filteredSongs => next(null, { filteredSongs, songs }))
  246. .catch(next);
  247. }
  248. ],
  249. (err, response) => {
  250. // console.log(222, err, payload);
  251. if (err && err !== true) {
  252. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Some error has occurred.", err.message);
  253. reject(new Error(err.message));
  254. } else {
  255. resolve({ songs: response.filteredSongs ? response.filteredSongs.youtubeIds : response.songs });
  256. }
  257. }
  258. );
  259. });
  260. }
  261. /**
  262. * Returns a a page from a YouTube playlist. Is used internally by GET_PLAYLIST.
  263. *
  264. * @param {object} payload - object that contains the payload
  265. * @param {boolean} payload.playlistId - the playlist id to get videos from
  266. * @param {boolean} payload.nextPageToken - the nextPageToken to use
  267. * @param {string} payload.url - the url of the YouTube playlist
  268. * @returns {Promise} - returns promise (reject, resolve)
  269. */
  270. GET_PLAYLIST_PAGE(payload) {
  271. return new Promise((resolve, reject) => {
  272. const params = {
  273. part: "contentDetails",
  274. playlistId: payload.playlistId,
  275. key: config.get("apis.youtube.key"),
  276. maxResults: 50
  277. };
  278. if (payload.nextPageToken) params.pageToken = payload.nextPageToken;
  279. YouTubeModule.rateLimiter.continue().then(() => {
  280. YouTubeModule.rateLimiter.restart();
  281. YouTubeModule.axios
  282. .get("https://www.googleapis.com/youtube/v3/playlistItems", {
  283. params,
  284. timeout: YouTubeModule.requestTimeout,
  285. raxConfig: {
  286. onRetryAttempt: err => {
  287. const cfg = rax.getConfig(err);
  288. YouTubeModule.log(
  289. "ERROR",
  290. "GET_PLAYLIST_PAGE",
  291. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  292. );
  293. }
  294. }
  295. })
  296. .then(res => {
  297. if (res.data.err) {
  298. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${res.data.error.message}`);
  299. return reject(new Error("An error has occured. Please try again later."));
  300. }
  301. const songs = res.data.items;
  302. if (res.data.nextPageToken) return resolve({ nextPageToken: res.data.nextPageToken, songs });
  303. return resolve({ songs });
  304. })
  305. .catch(err => {
  306. // console.log(333, err, payload);
  307. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${err.message}`);
  308. if (err.message === "Request failed with status code 404") {
  309. return reject(new Error("Playlist not found. Is the playlist public/unlisted?"));
  310. }
  311. return reject(new Error("An error has occured. Please try again later."));
  312. });
  313. });
  314. });
  315. }
  316. /**
  317. * Filters a list of YouTube videos so that they only contains videos with music. Is used internally by GET_PLAYLIST
  318. *
  319. * @param {object} payload - object that contains the payload
  320. * @param {Array} payload.videoIds - an array of YouTube videoIds to filter through
  321. * @param {Array} payload.page - the current page/set of video's to get, starting at 0. If left null, 0 is assumed. Will recurse.
  322. * @returns {Promise} - returns promise (reject, resolve)
  323. */
  324. FILTER_MUSIC_VIDEOS(payload) {
  325. return new Promise((resolve, reject) => {
  326. const page = payload.page ? payload.page : 0;
  327. const videosPerPage = 50;
  328. const localVideoIds = payload.videoIds.splice(page * 50, videosPerPage);
  329. if (localVideoIds.length === 0) {
  330. return resolve({ videoIds: [] });
  331. }
  332. const params = {
  333. part: "topicDetails",
  334. id: localVideoIds.join(","),
  335. key: config.get("apis.youtube.key"),
  336. maxResults: videosPerPage
  337. };
  338. return YouTubeModule.rateLimiter.continue().then(() => {
  339. YouTubeModule.rateLimiter.restart();
  340. YouTubeModule.axios
  341. .get("https://www.googleapis.com/youtube/v3/videos", {
  342. params,
  343. timeout: YouTubeModule.requestTimeout,
  344. raxConfig: {
  345. onRetryAttempt: err => {
  346. const cfg = rax.getConfig(err);
  347. YouTubeModule.log(
  348. "ERROR",
  349. "FILTER_MUSIC_VIDEOS",
  350. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  351. );
  352. }
  353. }
  354. })
  355. .then(res => {
  356. if (res.data.err) {
  357. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${res.data.error.message}`);
  358. return reject(new Error("An error has occured. Please try again later."));
  359. }
  360. const youtubeIds = [];
  361. res.data.items.forEach(item => {
  362. const youtubeId = item.id;
  363. if (!item.topicDetails) return;
  364. if (item.topicDetails.topicCategories.indexOf("https://en.wikipedia.org/wiki/Music") !== -1)
  365. youtubeIds.push(youtubeId);
  366. });
  367. return YouTubeModule.runJob(
  368. "FILTER_MUSIC_VIDEOS",
  369. { videoIds: payload.videoIds, page: page + 1 },
  370. this
  371. )
  372. .then(result => resolve({ youtubeIds: youtubeIds.concat(result.youtubeIds) }))
  373. .catch(err => reject(err));
  374. })
  375. .catch(err => {
  376. // console.log(444, err, payload);
  377. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${err.message}`);
  378. return reject(new Error("Failed to find playlist from YouTube"));
  379. });
  380. });
  381. });
  382. }
  383. }
  384. export default new _YouTubeModule();