youtube.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. import async from "async";
  2. import config from "config";
  3. import * as rax from "retry-axios";
  4. import axios from "axios";
  5. import CoreClass from "../core";
  6. class RateLimitter {
  7. /**
  8. * Constructor
  9. *
  10. * @param {number} timeBetween - The time between each allowed YouTube request
  11. */
  12. constructor(timeBetween) {
  13. this.dateStarted = Date.now();
  14. this.timeBetween = timeBetween;
  15. }
  16. /**
  17. * Returns a promise that resolves whenever the ratelimit of a YouTube request is done
  18. *
  19. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  20. */
  21. continue() {
  22. return new Promise(resolve => {
  23. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  24. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  25. });
  26. }
  27. /**
  28. * Restart the rate limit timer
  29. */
  30. restart() {
  31. this.dateStarted = Date.now();
  32. }
  33. }
  34. let YouTubeModule;
  35. class _YouTubeModule extends CoreClass {
  36. // eslint-disable-next-line require-jsdoc
  37. constructor() {
  38. super("youtube", {
  39. concurrency: 1,
  40. priorities: {
  41. GET_PLAYLIST: 11
  42. }
  43. });
  44. YouTubeModule = this;
  45. }
  46. /**
  47. * Initialises the activities module
  48. *
  49. * @returns {Promise} - returns promise (reject, resolve)
  50. */
  51. initialize() {
  52. return new Promise(resolve => {
  53. this.rateLimiter = new RateLimitter(config.get("apis.youtube.rateLimit"));
  54. this.requestTimeout = config.get("apis.youtube.requestTimeout");
  55. this.axios = axios.create();
  56. this.axios.defaults.raxConfig = {
  57. instance: this.axios,
  58. retry: config.get("apis.youtube.retryAmount"),
  59. noResponseRetries: config.get("apis.youtube.retryAmount")
  60. };
  61. rax.attach(this.axios);
  62. resolve();
  63. });
  64. }
  65. /**
  66. * Fetches a list of songs from Youtube's API
  67. *
  68. * @param {object} payload - object that contains the payload
  69. * @param {string} payload.query - the query we'll pass to youtubes api
  70. * @param {string} payload.pageToken - (optional) if this exists, will search search youtube for a specific page reference
  71. * @returns {Promise} - returns promise (reject, resolve)
  72. */
  73. SEARCH(payload) {
  74. const params = {
  75. part: "snippet",
  76. q: payload.query,
  77. key: config.get("apis.youtube.key"),
  78. type: "video",
  79. maxResults: 10
  80. };
  81. if (payload.pageToken) params.pageToken = payload.pageToken;
  82. return new Promise((resolve, reject) =>
  83. YouTubeModule.rateLimiter.continue().then(() => {
  84. YouTubeModule.rateLimiter.restart();
  85. YouTubeModule.axios
  86. .get("https://www.googleapis.com/youtube/v3/search", {
  87. params,
  88. raxConfig: {
  89. onRetryAttempt: err => {
  90. const cfg = rax.getConfig(err);
  91. YouTubeModule.log(
  92. "ERROR",
  93. "SEARCH",
  94. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  95. );
  96. }
  97. }
  98. })
  99. .then(res => {
  100. if (res.data.err) {
  101. YouTubeModule.log("ERROR", "SEARCH", `${res.data.error.message}`);
  102. return reject(new Error("An error has occured. Please try again later."));
  103. }
  104. return resolve(res.data);
  105. })
  106. .catch(err => {
  107. YouTubeModule.log("ERROR", "SEARCH", `${err.message}`);
  108. return reject(new Error("An error has occured. Please try again later."));
  109. });
  110. })
  111. );
  112. }
  113. /**
  114. * Gets the details of a song using the YouTube API
  115. *
  116. * @param {object} payload - object that contains the payload
  117. * @param {string} payload.youtubeId - the YouTube API id of the song
  118. * @returns {Promise} - returns promise (reject, resolve)
  119. */
  120. GET_SONG(payload) {
  121. return new Promise((resolve, reject) => {
  122. const params = {
  123. part: "snippet,contentDetails,statistics,status",
  124. id: payload.youtubeId,
  125. key: config.get("apis.youtube.key")
  126. };
  127. if (payload.pageToken) params.pageToken = payload.pageToken;
  128. YouTubeModule.rateLimiter.continue().then(() => {
  129. YouTubeModule.rateLimiter.restart();
  130. YouTubeModule.axios
  131. .get("https://www.googleapis.com/youtube/v3/videos", {
  132. params,
  133. timeout: YouTubeModule.requestTimeout,
  134. raxConfig: {
  135. onRetryAttempt: err => {
  136. const cfg = rax.getConfig(err);
  137. YouTubeModule.log(
  138. "ERROR",
  139. "GET_SONG",
  140. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  141. );
  142. }
  143. }
  144. })
  145. .then(res => {
  146. if (res.data.error) {
  147. YouTubeModule.log("ERROR", "GET_SONG", `${res.data.error.message}`);
  148. return reject(new Error("An error has occured. Please try again later."));
  149. }
  150. if (res.data.items[0] === undefined)
  151. return reject(
  152. new Error("The specified video does not exist or cannot be publicly accessed.")
  153. );
  154. // TODO Clean up duration converter
  155. let dur = res.data.items[0].contentDetails.duration;
  156. dur = dur.replace("PT", "");
  157. let duration = 0;
  158. dur = dur.replace(/([\d]*)H/, (v, v2) => {
  159. v2 = Number(v2);
  160. duration = v2 * 60 * 60;
  161. return "";
  162. });
  163. dur = dur.replace(/([\d]*)M/, (v, v2) => {
  164. v2 = Number(v2);
  165. duration += v2 * 60;
  166. return "";
  167. });
  168. // eslint-disable-next-line no-unused-vars
  169. dur = dur.replace(/([\d]*)S/, (v, v2) => {
  170. v2 = Number(v2);
  171. duration += v2;
  172. return "";
  173. });
  174. const song = {
  175. youtubeId: res.data.items[0].id,
  176. title: res.data.items[0].snippet.title,
  177. thumbnail: res.data.items[0].snippet.thumbnails.default.url,
  178. duration
  179. };
  180. return resolve({ song });
  181. })
  182. .catch(err => {
  183. YouTubeModule.log("ERROR", "GET_SONG", `${err.message}`);
  184. return reject(new Error("An error has occured. Please try again later."));
  185. });
  186. });
  187. });
  188. }
  189. /**
  190. * Returns an array of songs taken from a YouTube playlist
  191. *
  192. * @param {object} payload - object that contains the payload
  193. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the playlist
  194. * @param {string} payload.url - the url of the YouTube playlist
  195. * @returns {Promise} - returns promise (reject, resolve)
  196. */
  197. GET_PLAYLIST(payload) {
  198. return new Promise((resolve, reject) => {
  199. const regex = new RegExp(`[\\?&]list=([^&#]*)`);
  200. const splitQuery = regex.exec(payload.url);
  201. if (!splitQuery) {
  202. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Invalid YouTube playlist URL query.");
  203. return reject(new Error("Invalid playlist URL."));
  204. }
  205. const playlistId = splitQuery[1];
  206. return async.waterfall(
  207. [
  208. next => {
  209. let songs = [];
  210. let nextPageToken = "";
  211. async.whilst(
  212. next => {
  213. YouTubeModule.log(
  214. "INFO",
  215. `Getting playlist progress for job (${this.toString()}): ${
  216. songs.length
  217. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  218. );
  219. next(null, nextPageToken !== undefined);
  220. },
  221. next => {
  222. // Add 250ms delay between each job request
  223. setTimeout(() => {
  224. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  225. .then(response => {
  226. songs = songs.concat(response.songs);
  227. nextPageToken = response.nextPageToken;
  228. next();
  229. })
  230. .catch(err => next(err));
  231. }, 250);
  232. },
  233. err => next(err, songs)
  234. );
  235. },
  236. (songs, next) =>
  237. next(
  238. null,
  239. songs.map(song => song.contentDetails.videoId)
  240. ),
  241. (songs, next) => {
  242. if (!payload.musicOnly) return next(true, { songs });
  243. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  244. .then(filteredSongs => next(null, { filteredSongs, songs }))
  245. .catch(next);
  246. }
  247. ],
  248. (err, response) => {
  249. if (err && err !== true) {
  250. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Some error has occurred.", err.message);
  251. reject(new Error(err.message));
  252. } else {
  253. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  254. }
  255. }
  256. );
  257. });
  258. }
  259. /**
  260. * Returns a a page from a YouTube playlist. Is used internally by GET_PLAYLIST.
  261. *
  262. * @param {object} payload - object that contains the payload
  263. * @param {boolean} payload.playlistId - the playlist id to get videos from
  264. * @param {boolean} payload.nextPageToken - the nextPageToken to use
  265. * @param {string} payload.url - the url of the YouTube playlist
  266. * @returns {Promise} - returns promise (reject, resolve)
  267. */
  268. GET_PLAYLIST_PAGE(payload) {
  269. return new Promise((resolve, reject) => {
  270. const params = {
  271. part: "contentDetails",
  272. playlistId: payload.playlistId,
  273. key: config.get("apis.youtube.key"),
  274. maxResults: 50
  275. };
  276. if (payload.nextPageToken) params.pageToken = payload.nextPageToken;
  277. YouTubeModule.rateLimiter.continue().then(() => {
  278. YouTubeModule.rateLimiter.restart();
  279. YouTubeModule.axios
  280. .get("https://www.googleapis.com/youtube/v3/playlistItems", {
  281. params,
  282. timeout: YouTubeModule.requestTimeout,
  283. raxConfig: {
  284. onRetryAttempt: err => {
  285. const cfg = rax.getConfig(err);
  286. YouTubeModule.log(
  287. "ERROR",
  288. "GET_PLAYLIST_PAGE",
  289. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  290. );
  291. }
  292. }
  293. })
  294. .then(res => {
  295. if (res.data.err) {
  296. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${res.data.error.message}`);
  297. return reject(new Error("An error has occured. Please try again later."));
  298. }
  299. const songs = res.data.items;
  300. if (res.data.nextPageToken) return resolve({ nextPageToken: res.data.nextPageToken, songs });
  301. return resolve({ songs });
  302. })
  303. .catch(err => {
  304. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${err.message}`);
  305. if (err.message === "Request failed with status code 404") {
  306. return reject(new Error("Playlist not found. Is the playlist public/unlisted?"));
  307. }
  308. return reject(new Error("An error has occured. Please try again later."));
  309. });
  310. });
  311. });
  312. }
  313. /**
  314. * Filters a list of YouTube videos so that they only contains videos with music. Is used internally by GET_PLAYLIST
  315. *
  316. * @param {object} payload - object that contains the payload
  317. * @param {Array} payload.videoIds - an array of YouTube videoIds to filter through
  318. * @param {Array} payload.page - the current page/set of video's to get, starting at 0. If left null, 0 is assumed. Will recurse.
  319. * @returns {Promise} - returns promise (reject, resolve)
  320. */
  321. FILTER_MUSIC_VIDEOS(payload) {
  322. return new Promise((resolve, reject) => {
  323. const page = payload.page ? payload.page : 0;
  324. const videosPerPage = 50;
  325. const localVideoIds = payload.videoIds.splice(page * 50, videosPerPage);
  326. if (localVideoIds.length === 0) {
  327. return resolve({ videoIds: [] });
  328. }
  329. const params = {
  330. part: "topicDetails",
  331. id: localVideoIds.join(","),
  332. key: config.get("apis.youtube.key"),
  333. maxResults: videosPerPage
  334. };
  335. return YouTubeModule.rateLimiter.continue().then(() => {
  336. YouTubeModule.rateLimiter.restart();
  337. YouTubeModule.axios
  338. .get("https://www.googleapis.com/youtube/v3/videos", {
  339. params,
  340. timeout: YouTubeModule.requestTimeout,
  341. raxConfig: {
  342. onRetryAttempt: err => {
  343. const cfg = rax.getConfig(err);
  344. YouTubeModule.log(
  345. "ERROR",
  346. "FILTER_MUSIC_VIDEOS",
  347. `Attempt #${cfg.currentRetryAttempt}. Error: ${err.message}`
  348. );
  349. }
  350. }
  351. })
  352. .then(res => {
  353. if (res.data.err) {
  354. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${res.data.error.message}`);
  355. return reject(new Error("An error has occured. Please try again later."));
  356. }
  357. const videoIds = [];
  358. res.data.items.forEach(item => {
  359. const videoId = item.id;
  360. if (!item.topicDetails) return;
  361. if (item.topicDetails.topicCategories.indexOf("https://en.wikipedia.org/wiki/Music") !== -1)
  362. videoIds.push(videoId);
  363. });
  364. return YouTubeModule.runJob(
  365. "FILTER_MUSIC_VIDEOS",
  366. { videoIds: payload.videoIds, page: page + 1 },
  367. this
  368. )
  369. .then(result => resolve({ videoIds: videoIds.concat(result.videoIds) }))
  370. .catch(err => reject(err));
  371. })
  372. .catch(err => {
  373. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${err.message}`);
  374. return reject(new Error("Failed to find playlist from YouTube"));
  375. });
  376. });
  377. });
  378. }
  379. }
  380. export default new _YouTubeModule();