youtube.js 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727
  1. import mongoose from "mongoose";
  2. import async from "async";
  3. import config from "config";
  4. import * as rax from "retry-axios";
  5. import axios from "axios";
  6. import CoreClass from "../core";
  7. class RateLimitter {
  8. /**
  9. * Constructor
  10. *
  11. * @param {number} timeBetween - The time between each allowed YouTube request
  12. */
  13. constructor(timeBetween) {
  14. this.dateStarted = Date.now();
  15. this.timeBetween = timeBetween;
  16. }
  17. /**
  18. * Returns a promise that resolves whenever the ratelimit of a YouTube request is done
  19. *
  20. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  21. */
  22. continue() {
  23. return new Promise(resolve => {
  24. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  25. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  26. });
  27. }
  28. /**
  29. * Restart the rate limit timer
  30. */
  31. restart() {
  32. this.dateStarted = Date.now();
  33. }
  34. }
  35. let YouTubeModule;
  36. let CacheModule;
  37. let DBModule;
  38. let MediaModule;
  39. let SongsModule;
  40. let StationsModule;
  41. let PlaylistsModule;
  42. let WSModule;
  43. const isQuotaExceeded = apiCalls => {
  44. const reversedApiCalls = apiCalls.slice().reverse();
  45. const quotas = config.get("apis.youtube.quotas").slice();
  46. const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
  47. let quotaExceeded = false;
  48. sortedQuotas.forEach(quota => {
  49. let quotaUsed = 0;
  50. let dateCutoff = null;
  51. if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date() - 1000 * 60;
  52. else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date() - 1000 * 100;
  53. else if (quota.type === "QUERIES_PER_DAY") {
  54. // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
  55. dateCutoff = new Date();
  56. dateCutoff.setUTCMilliseconds(0);
  57. dateCutoff.setUTCSeconds(0);
  58. dateCutoff.setUTCMinutes(0);
  59. dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
  60. dateCutoff.setUTCHours(0);
  61. }
  62. reversedApiCalls.forEach(apiCall => {
  63. if (apiCall.date >= dateCutoff) quotaUsed += apiCall.quotaCost;
  64. });
  65. if (quotaUsed >= quota.limit) {
  66. quotaExceeded = true;
  67. }
  68. });
  69. return quotaExceeded;
  70. };
  71. class _YouTubeModule extends CoreClass {
  72. // eslint-disable-next-line require-jsdoc
  73. constructor() {
  74. super("youtube", {
  75. concurrency: 10,
  76. priorities: {
  77. GET_PLAYLIST: 11
  78. }
  79. });
  80. YouTubeModule = this;
  81. }
  82. /**
  83. * Initialises the activities module
  84. *
  85. * @returns {Promise} - returns promise (reject, resolve)
  86. */
  87. async initialize() {
  88. CacheModule = this.moduleManager.modules.cache;
  89. DBModule = this.moduleManager.modules.db;
  90. MediaModule = this.moduleManager.modules.media;
  91. SongsModule = this.moduleManager.modules.songs;
  92. StationsModule = this.moduleManager.modules.stations;
  93. PlaylistsModule = this.moduleManager.modules.playlists;
  94. WSModule = this.moduleManager.modules.ws;
  95. this.youtubeApiRequestModel = this.YoutubeApiRequestModel = await DBModule.runJob("GET_MODEL", {
  96. modelName: "youtubeApiRequest"
  97. });
  98. this.youtubeVideoModel = this.YoutubeVideoModel = await DBModule.runJob("GET_MODEL", {
  99. modelName: "youtubeVideo"
  100. });
  101. return new Promise(resolve => {
  102. CacheModule.runJob("SUB", {
  103. channel: "youtube.removeYoutubeApiRequest",
  104. cb: requestId => {
  105. WSModule.runJob("EMIT_TO_ROOM", {
  106. room: `view-api-request.${requestId}`,
  107. args: ["event:youtubeApiRequest.removed"]
  108. });
  109. WSModule.runJob("EMIT_TO_ROOM", {
  110. room: "admin.youtube",
  111. args: ["event:admin.youtubeApiRequest.removed", { data: { requestId } }]
  112. });
  113. }
  114. });
  115. CacheModule.runJob("SUB", {
  116. channel: "youtube.removeVideos",
  117. cb: videoIds => {
  118. const videos = Array.isArray(videoIds) ? videoIds : [videoIds];
  119. videos.forEach(videoId => {
  120. WSModule.runJob("EMIT_TO_ROOM", {
  121. room: `view-youtube-video.${videoId}`,
  122. args: ["event:youtubeVideo.removed"]
  123. });
  124. WSModule.runJob("EMIT_TO_ROOM", {
  125. room: "admin.youtubeVideos",
  126. args: ["event:admin.youtubeVideo.removed", { data: { videoId } }]
  127. });
  128. WSModule.runJob("EMIT_TO_ROOMS", {
  129. rooms: ["import-album", "edit-songs"],
  130. args: ["event:admin.youtubeVideo.removed", { videoId }]
  131. });
  132. });
  133. }
  134. });
  135. this.rateLimiter = new RateLimitter(config.get("apis.youtube.rateLimit"));
  136. this.requestTimeout = config.get("apis.youtube.requestTimeout");
  137. this.axios = axios.create();
  138. this.axios.defaults.raxConfig = {
  139. instance: this.axios,
  140. retry: config.get("apis.youtube.retryAmount"),
  141. noResponseRetries: config.get("apis.youtube.retryAmount")
  142. };
  143. rax.attach(this.axios);
  144. this.youtubeApiRequestModel
  145. .find(
  146. { date: { $gte: new Date() - 2 * 24 * 60 * 60 * 1000 } },
  147. { date: true, quotaCost: true, _id: false }
  148. )
  149. .sort({ date: 1 })
  150. .exec((err, youtubeApiRequests) => {
  151. if (err) console.log("Couldn't load YouTube API requests.");
  152. else {
  153. this.apiCalls = youtubeApiRequests;
  154. resolve();
  155. }
  156. });
  157. });
  158. }
  159. /**
  160. * Fetches a list of songs from Youtube's API
  161. *
  162. * @param {object} payload - object that contains the payload
  163. * @param {string} payload.query - the query we'll pass to youtubes api
  164. * @param {string} payload.pageToken - (optional) if this exists, will search search youtube for a specific page reference
  165. * @returns {Promise} - returns promise (reject, resolve)
  166. */
  167. SEARCH(payload) {
  168. const params = {
  169. part: "snippet",
  170. q: payload.query,
  171. type: "video",
  172. maxResults: 10
  173. };
  174. if (payload.pageToken) params.pageToken = payload.pageToken;
  175. return new Promise((resolve, reject) => {
  176. YouTubeModule.runJob(
  177. "API_SEARCH",
  178. {
  179. params
  180. },
  181. this
  182. )
  183. .then(({ response }) => {
  184. const { data } = response;
  185. return resolve(data);
  186. })
  187. .catch(err => {
  188. YouTubeModule.log("ERROR", "SEARCH", `${err.message}`);
  189. return reject(new Error("An error has occured. Please try again later."));
  190. });
  191. });
  192. }
  193. /**
  194. * Returns details about the YouTube quota usage
  195. *
  196. * @param {object} payload - object that contains the payload
  197. * @param {string} payload.fromDate - date to select requests up to
  198. * @returns {Promise} - returns promise (reject, resolve)
  199. */
  200. GET_QUOTA_STATUS(payload) {
  201. return new Promise((resolve, reject) => {
  202. const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
  203. YouTubeModule.youtubeApiRequestModel
  204. .find(
  205. { date: { $gte: fromDate - 2 * 24 * 60 * 60 * 1000, $lte: fromDate } },
  206. { date: true, quotaCost: true, _id: false }
  207. )
  208. .sort({ date: 1 })
  209. .exec((err, youtubeApiRequests) => {
  210. if (err) reject(new Error("Couldn't load YouTube API requests."));
  211. else {
  212. const reversedApiCalls = youtubeApiRequests.slice().reverse();
  213. const quotas = config.get("apis.youtube.quotas").slice();
  214. const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
  215. const status = {};
  216. sortedQuotas.forEach(quota => {
  217. status[quota.type] = {
  218. title: quota.title,
  219. quotaUsed: 0,
  220. limit: quota.limit,
  221. quotaExceeded: false
  222. };
  223. let dateCutoff = null;
  224. if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date(fromDate) - 1000 * 60;
  225. else if (quota.type === "QUERIES_PER_100_SECONDS")
  226. dateCutoff = new Date(fromDate) - 1000 * 100;
  227. else if (quota.type === "QUERIES_PER_DAY") {
  228. // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
  229. dateCutoff = new Date(fromDate);
  230. dateCutoff.setUTCMilliseconds(0);
  231. dateCutoff.setUTCSeconds(0);
  232. dateCutoff.setUTCMinutes(0);
  233. dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
  234. dateCutoff.setUTCHours(0);
  235. }
  236. reversedApiCalls.forEach(apiCall => {
  237. if (apiCall.date >= dateCutoff) status[quota.type].quotaUsed += apiCall.quotaCost;
  238. });
  239. if (status[quota.type].quotaUsed >= quota.limit && !status[quota.type].quotaExceeded)
  240. status[quota.type].quotaExceeded = true;
  241. });
  242. resolve({ status });
  243. }
  244. });
  245. });
  246. }
  247. /**
  248. * Returns YouTube quota chart data
  249. *
  250. * @param {object} payload - object that contains the payload
  251. * @param {string} payload.timePeriod - either hours or days
  252. * @param {string} payload.startDate - beginning date
  253. * @param {string} payload.endDate - end date
  254. * @param {string} payload.dataType - either usage or count
  255. * @returns {Promise} - returns promise (reject, resolve)
  256. */
  257. GET_QUOTA_CHART_DATA(payload) {
  258. return new Promise((resolve, reject) => {
  259. const { timePeriod, startDate, endDate, dataType } = payload;
  260. // const timePeriod = "hours";
  261. // const startDate = new Date("2022-05-20 00:00:00");
  262. // const endDate = new Date("2022-05-21 00:00:00");
  263. // const timePeriod = "days";
  264. // const startDate = new Date("2022-05-15 00:00:00");
  265. // const endDate = new Date("2022-05-21 00:00:00");
  266. // const endDate = new Date("2022-05-30 00:00:00");
  267. // const dataType = "usage";
  268. // const dataType = "count";
  269. async.waterfall(
  270. [
  271. next => {
  272. let timeRanges = [];
  273. if (timePeriod === "hours") {
  274. startDate.setMinutes(0, 0, 0);
  275. endDate.setMinutes(0, 0, 0);
  276. const lastDate = new Date(startDate);
  277. do {
  278. const newDate = new Date(lastDate.getTime() + 1000 * 60 * 60);
  279. timeRanges.push({
  280. startDate: new Date(lastDate),
  281. endDate: newDate
  282. });
  283. lastDate.setTime(newDate.getTime());
  284. } while (lastDate.getTime() < endDate.getTime());
  285. if (timeRanges.length === 0 || timeRanges.length > 24)
  286. return next("No valid time ranges specified.");
  287. timeRanges = timeRanges.map(timeRange => ({
  288. ...timeRange,
  289. label: `${timeRange.startDate.getHours().toString().padStart(2, "0")}:00`
  290. }));
  291. } else if (timePeriod === "days") {
  292. startDate.setHours(0, 0, 0, 0);
  293. endDate.setHours(0, 0, 0, 0);
  294. const lastDate = new Date(startDate);
  295. do {
  296. const newDate = new Date(lastDate.getTime() + 1000 * 60 * 60 * 24);
  297. timeRanges.push({
  298. startDate: new Date(lastDate),
  299. endDate: newDate
  300. });
  301. lastDate.setTime(newDate.getTime());
  302. } while (lastDate.getTime() < endDate.getTime());
  303. if (timeRanges.length === 0 || timeRanges.length > 31)
  304. return next("No valid time ranges specified.");
  305. const days = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
  306. if (timeRanges.length <= 7)
  307. timeRanges = timeRanges.map(timeRange => ({
  308. ...timeRange,
  309. label: days[timeRange.startDate.getDay()]
  310. }));
  311. else
  312. timeRanges = timeRanges.map(timeRange => ({
  313. ...timeRange,
  314. label: `${timeRange.startDate.getDate().toString().padStart(2, "0")}-${(
  315. timeRange.startDate.getMonth() + 1
  316. )
  317. .toString()
  318. .padStart(2, "0")}`
  319. }));
  320. }
  321. return next(null, timeRanges);
  322. },
  323. (timeRanges, next) => {
  324. YouTubeModule.youtubeApiRequestModel
  325. .find({
  326. date: { $gte: startDate, $lte: endDate }
  327. })
  328. .sort({ date: 1 })
  329. .exec((err, youtubeApiRequests) => {
  330. next(err, timeRanges, youtubeApiRequests);
  331. });
  332. },
  333. (timeRanges, youtubeApiRequests, next) => {
  334. const regex = /https:\/\/www\.googleapis\.com\/youtube\/v3\/(.+)/;
  335. const requestTypes = Object.fromEntries(
  336. youtubeApiRequests
  337. .map(youtubeApiRequest => regex.exec(youtubeApiRequest.url)[1])
  338. .filter((requestType, index, array) => array.indexOf(requestType) === index)
  339. .map(requestType => [requestType, 0])
  340. );
  341. timeRanges = timeRanges.map(timeRange => ({
  342. ...timeRange,
  343. data: { total: 0, ...requestTypes }
  344. }));
  345. youtubeApiRequests.forEach(youtubeApiRequest => {
  346. timeRanges.forEach(timeRange => {
  347. if (
  348. timeRange.startDate <= youtubeApiRequest.date &&
  349. timeRange.endDate >= youtubeApiRequest.date
  350. ) {
  351. const requestType = regex.exec(youtubeApiRequest.url)[1];
  352. if (!timeRange.data[requestType]) timeRange.data[requestType] = 0;
  353. if (dataType === "usage") {
  354. timeRange.data[requestType] += youtubeApiRequest.quotaCost;
  355. timeRange.data.total += youtubeApiRequest.quotaCost;
  356. } else if (dataType === "count") {
  357. timeRange.data[requestType] += 1;
  358. timeRange.data.total += 1;
  359. }
  360. }
  361. });
  362. });
  363. next(null, timeRanges);
  364. },
  365. (timeRanges, next) => {
  366. const chartData = {};
  367. chartData.labels = timeRanges.map(timeRange => timeRange.label);
  368. const datasetTypes = Object.keys(timeRanges[0].data);
  369. const colors = {
  370. total: "rgb(2, 166, 242)",
  371. videos: "rgb(166, 2, 242)",
  372. search: "rgb(242, 2, 166)",
  373. channels: "rgb(2, 242, 166)",
  374. playlistItems: "rgb(242, 166, 2)"
  375. };
  376. chartData.datasets = datasetTypes.map(datasetType => {
  377. let label;
  378. switch (datasetType) {
  379. case "total":
  380. label = "Total";
  381. break;
  382. case "videos":
  383. label = "Videos";
  384. break;
  385. case "search":
  386. label = "Search";
  387. break;
  388. case "playlistItems":
  389. label = "Playlist Items";
  390. break;
  391. default:
  392. label = datasetType;
  393. }
  394. return {
  395. label,
  396. borderColor: colors[datasetType],
  397. backgroundColor: colors[datasetType],
  398. data: timeRanges.map(timeRange => timeRange.data[datasetType])
  399. };
  400. });
  401. next(null, chartData);
  402. }
  403. ],
  404. (err, chartData) => {
  405. if (err) reject(err);
  406. else resolve(chartData);
  407. }
  408. );
  409. });
  410. }
  411. /**
  412. * Gets the id of the channel upload playlist
  413. *
  414. * @param {object} payload - object that contains the payload
  415. * @param {string} payload.id - the id of the YouTube channel. Optional: can be left out if specifying a username.
  416. * @param {string} payload.username - the username of the YouTube channel. Only gets used if no id is specified.
  417. * @returns {Promise} - returns promise (reject, resolve)
  418. */
  419. GET_CHANNEL_UPLOADS_PLAYLIST_ID(payload) {
  420. return new Promise((resolve, reject) => {
  421. const params = {
  422. part: "id,contentDetails"
  423. };
  424. if (payload.id) params.id = payload.id;
  425. else params.forUsername = payload.username;
  426. YouTubeModule.runJob(
  427. "API_GET_CHANNELS",
  428. {
  429. params
  430. },
  431. this
  432. )
  433. .then(({ response }) => {
  434. const { data } = response;
  435. if (data.pageInfo.totalResults === 0) return reject(new Error("Channel not found."));
  436. const playlistId = data.items[0].contentDetails.relatedPlaylists.uploads;
  437. return resolve({ playlistId });
  438. })
  439. .catch(err => {
  440. YouTubeModule.log("ERROR", "GET_CHANNEL_UPLOADS_PLAYLIST_ID", `${err.message}`);
  441. if (err.message === "Request failed with status code 404") {
  442. return reject(new Error("Channel not found. Is the channel public/unlisted?"));
  443. }
  444. return reject(new Error("An error has occured. Please try again later."));
  445. });
  446. });
  447. }
  448. /**
  449. * Gets the id of the channel from the custom URL
  450. *
  451. * @param {object} payload - object that contains the payload
  452. * @param {string} payload.customUrl - the customUrl of the YouTube channel
  453. * @returns {Promise} - returns promise (reject, resolve)
  454. */
  455. GET_CHANNEL_ID_FROM_CUSTOM_URL(payload) {
  456. return new Promise((resolve, reject) => {
  457. async.waterfall(
  458. [
  459. next => {
  460. const params = {
  461. part: "snippet",
  462. type: "channel",
  463. maxResults: 50
  464. };
  465. params.q = payload.customUrl;
  466. YouTubeModule.runJob(
  467. "API_SEARCH",
  468. {
  469. params
  470. },
  471. this
  472. )
  473. .then(({ response }) => {
  474. const { data } = response;
  475. if (data.pageInfo.totalResults === 0) return next("Channel not found.");
  476. const channelIds = data.items.map(item => item.id.channelId);
  477. return next(null, channelIds);
  478. })
  479. .catch(err => {
  480. next(err);
  481. });
  482. },
  483. (channelIds, next) => {
  484. const params = {
  485. part: "snippet",
  486. id: channelIds.join(","),
  487. maxResults: 50
  488. };
  489. YouTubeModule.runJob(
  490. "API_GET_CHANNELS",
  491. {
  492. params
  493. },
  494. this
  495. )
  496. .then(({ response }) => {
  497. const { data } = response;
  498. if (data.pageInfo.totalResults === 0) return next("Channel not found.");
  499. let channelId = null;
  500. data.items.forEach(item => {
  501. if (
  502. item.snippet.customUrl &&
  503. item.snippet.customUrl.toLowerCase() === payload.customUrl.toLowerCase()
  504. ) {
  505. channelId = item.id;
  506. }
  507. });
  508. if (!channelId) return next("Channel not found.");
  509. return next(null, channelId);
  510. })
  511. .catch(err => {
  512. next(err);
  513. });
  514. }
  515. ],
  516. (err, channelId) => {
  517. if (err) {
  518. YouTubeModule.log("ERROR", "GET_CHANNEL_ID_FROM_CUSTOM_URL", `${err.message}`);
  519. if (err.message === "Request failed with status code 404") {
  520. return reject(new Error("Channel not found. Is the channel public/unlisted?"));
  521. }
  522. return reject(new Error("An error has occured. Please try again later."));
  523. }
  524. return resolve({ channelId });
  525. }
  526. );
  527. });
  528. }
  529. /**
  530. * Returns an array of songs taken from a YouTube playlist
  531. *
  532. * @param {object} payload - object that contains the payload
  533. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the playlist
  534. * @param {string} payload.url - the url of the YouTube playlist
  535. * @returns {Promise} - returns promise (reject, resolve)
  536. */
  537. GET_PLAYLIST(payload) {
  538. return new Promise((resolve, reject) => {
  539. const regex = /[\\?&]list=([^&#]*)/;
  540. const splitQuery = regex.exec(payload.url);
  541. if (!splitQuery) {
  542. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Invalid YouTube playlist URL query.");
  543. reject(new Error("Invalid playlist URL."));
  544. return;
  545. }
  546. const playlistId = splitQuery[1];
  547. async.waterfall(
  548. [
  549. next => {
  550. let songs = [];
  551. let nextPageToken = "";
  552. async.whilst(
  553. next => {
  554. YouTubeModule.log(
  555. "INFO",
  556. `Getting playlist progress for job (${this.toString()}): ${
  557. songs.length
  558. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  559. );
  560. next(null, nextPageToken !== undefined);
  561. },
  562. next => {
  563. // Add 250ms delay between each job request
  564. setTimeout(() => {
  565. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  566. .then(response => {
  567. songs = songs.concat(response.songs);
  568. nextPageToken = response.nextPageToken;
  569. next();
  570. })
  571. .catch(err => next(err));
  572. }, 250);
  573. },
  574. err => next(err, songs)
  575. );
  576. },
  577. (songs, next) =>
  578. next(
  579. null,
  580. songs.map(song => song.contentDetails.videoId)
  581. ),
  582. (songs, next) => {
  583. if (!payload.musicOnly) return next(true, { songs });
  584. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  585. .then(filteredSongs => next(null, { filteredSongs, songs }))
  586. .catch(next);
  587. }
  588. ],
  589. (err, response) => {
  590. if (err && err !== true) {
  591. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Some error has occurred.", err.message);
  592. reject(new Error(err.message));
  593. } else {
  594. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  595. }
  596. }
  597. );
  598. });
  599. }
  600. /**
  601. * Returns a a page from a YouTube playlist. Is used internally by GET_PLAYLIST and GET_CHANNEL.
  602. *
  603. * @param {object} payload - object that contains the payload
  604. * @param {boolean} payload.playlistId - the playlist id to get videos from
  605. * @param {boolean} payload.nextPageToken - the nextPageToken to use
  606. * @param {string} payload.url - the url of the YouTube playlist
  607. * @returns {Promise} - returns promise (reject, resolve)
  608. */
  609. GET_PLAYLIST_PAGE(payload) {
  610. return new Promise((resolve, reject) => {
  611. const params = {
  612. part: "contentDetails",
  613. playlistId: payload.playlistId,
  614. maxResults: 50
  615. };
  616. if (payload.nextPageToken) params.pageToken = payload.nextPageToken;
  617. YouTubeModule.runJob(
  618. "API_GET_PLAYLIST_ITEMS",
  619. {
  620. params
  621. },
  622. this
  623. )
  624. .then(({ response }) => {
  625. const { data } = response;
  626. const songs = data.items;
  627. if (data.nextPageToken) return resolve({ nextPageToken: data.nextPageToken, songs });
  628. return resolve({ songs });
  629. })
  630. .catch(err => {
  631. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${err.message}`);
  632. if (err.message === "Request failed with status code 404") {
  633. return reject(new Error("Playlist not found. Is the playlist public/unlisted?"));
  634. }
  635. return reject(new Error("An error has occured. Please try again later."));
  636. });
  637. });
  638. }
  639. /**
  640. * Filters a list of YouTube videos so that they only contains videos with music. Is used internally by GET_PLAYLIST
  641. *
  642. * @param {object} payload - object that contains the payload
  643. * @param {Array} payload.videoIds - an array of YouTube videoIds to filter through
  644. * @param {Array} payload.page - the current page/set of video's to get, starting at 0. If left null, 0 is assumed. Will recurse.
  645. * @returns {Promise} - returns promise (reject, resolve)
  646. */
  647. FILTER_MUSIC_VIDEOS(payload) {
  648. return new Promise((resolve, reject) => {
  649. const page = payload.page ? payload.page : 0;
  650. const videosPerPage = 50;
  651. const localVideoIds = payload.videoIds.splice(page * 50, videosPerPage);
  652. if (localVideoIds.length === 0) {
  653. resolve({ videoIds: [] });
  654. return;
  655. }
  656. const params = {
  657. part: "topicDetails",
  658. id: localVideoIds.join(","),
  659. maxResults: videosPerPage
  660. };
  661. YouTubeModule.runJob("API_GET_VIDEOS", { params }, this)
  662. .then(({ response }) => {
  663. const { data } = response;
  664. const videoIds = [];
  665. data.items.forEach(item => {
  666. const videoId = item.id;
  667. if (!item.topicDetails) return;
  668. if (item.topicDetails.topicCategories.indexOf("https://en.wikipedia.org/wiki/Music") !== -1)
  669. videoIds.push(videoId);
  670. });
  671. return YouTubeModule.runJob(
  672. "FILTER_MUSIC_VIDEOS",
  673. { videoIds: payload.videoIds, page: page + 1 },
  674. this
  675. )
  676. .then(result => resolve({ videoIds: videoIds.concat(result.videoIds) }))
  677. .catch(err => reject(err));
  678. })
  679. .catch(err => {
  680. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${err.message}`);
  681. return reject(new Error("Failed to find playlist from YouTube"));
  682. });
  683. });
  684. }
  685. /**
  686. * Returns an array of songs taken from a YouTube channel
  687. *
  688. * @param {object} payload - object that contains the payload
  689. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the channel
  690. * @param {string} payload.url - the url of the YouTube channel
  691. * @returns {Promise} - returns promise (reject, resolve)
  692. */
  693. GET_CHANNEL(payload) {
  694. return new Promise((resolve, reject) => {
  695. const regex =
  696. /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/;
  697. const splitQuery = regex.exec(payload.url);
  698. if (!splitQuery) {
  699. YouTubeModule.log("ERROR", "GET_CHANNEL", "Invalid YouTube channel URL query.");
  700. reject(new Error("Invalid playlist URL."));
  701. return;
  702. }
  703. const channelId = splitQuery[1];
  704. const channelUsername = splitQuery[2];
  705. const channelCustomUrl = splitQuery[3];
  706. const channelUsernameOrCustomUrl = splitQuery[4];
  707. console.log(`Channel id: ${channelId}`);
  708. console.log(`Channel username: ${channelUsername}`);
  709. console.log(`Channel custom URL: ${channelCustomUrl}`);
  710. console.log(`Channel username or custom URL: ${channelUsernameOrCustomUrl}`);
  711. async.waterfall(
  712. [
  713. next => {
  714. const payload = {};
  715. if (channelId) payload.id = channelId;
  716. else if (channelUsername) payload.username = channelUsername;
  717. else return next(null, true, null);
  718. return YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", payload, this)
  719. .then(({ playlistId }) => {
  720. next(null, false, playlistId);
  721. })
  722. .catch(err => {
  723. if (err.message === "Channel not found. Is the channel public/unlisted?")
  724. next(null, true, null);
  725. else next(err);
  726. });
  727. },
  728. (getUsernameFromCustomUrl, playlistId, next) => {
  729. if (!getUsernameFromCustomUrl) return next(null, playlistId);
  730. const payload = {};
  731. if (channelCustomUrl) payload.customUrl = channelCustomUrl;
  732. else if (channelUsernameOrCustomUrl) payload.customUrl = channelUsernameOrCustomUrl;
  733. else return next("No proper URL provided.");
  734. return YouTubeModule.runJob("GET_CHANNEL_ID_FROM_CUSTOM_URL", payload, this)
  735. .then(({ channelId }) => {
  736. YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", { id: channelId }, this)
  737. .then(({ playlistId }) => {
  738. next(null, playlistId);
  739. })
  740. .catch(err => next(err));
  741. })
  742. .catch(err => next(err));
  743. },
  744. (playlistId, next) => {
  745. let songs = [];
  746. let nextPageToken = "";
  747. async.whilst(
  748. next => {
  749. YouTubeModule.log(
  750. "INFO",
  751. `Getting channel progress for job (${this.toString()}): ${
  752. songs.length
  753. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  754. );
  755. next(null, nextPageToken !== undefined);
  756. },
  757. next => {
  758. // Add 250ms delay between each job request
  759. setTimeout(() => {
  760. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  761. .then(response => {
  762. songs = songs.concat(response.songs);
  763. nextPageToken = response.nextPageToken;
  764. next();
  765. })
  766. .catch(err => next(err));
  767. }, 250);
  768. },
  769. err => next(err, songs)
  770. );
  771. },
  772. (songs, next) =>
  773. next(
  774. null,
  775. songs.map(song => song.contentDetails.videoId)
  776. ),
  777. (songs, next) => {
  778. if (!payload.musicOnly) return next(true, { songs });
  779. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  780. .then(filteredSongs => next(null, { filteredSongs, songs }))
  781. .catch(next);
  782. }
  783. ],
  784. (err, response) => {
  785. if (err && err !== true) {
  786. YouTubeModule.log("ERROR", "GET_CHANNEL", "Some error has occurred.", err.message);
  787. reject(new Error(err.message));
  788. } else {
  789. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  790. }
  791. }
  792. );
  793. });
  794. }
  795. /**
  796. * Perform YouTube API get videos request
  797. *
  798. * @param {object} payload - object that contains the payload
  799. * @param {object} payload.params - request parameters
  800. * @returns {Promise} - returns promise (reject, resolve)
  801. */
  802. API_GET_VIDEOS(payload) {
  803. return new Promise((resolve, reject) => {
  804. const { params } = payload;
  805. YouTubeModule.runJob(
  806. "API_CALL",
  807. {
  808. url: "https://www.googleapis.com/youtube/v3/videos",
  809. params: {
  810. key: config.get("apis.youtube.key"),
  811. ...params
  812. },
  813. quotaCost: 1
  814. },
  815. this
  816. )
  817. .then(response => {
  818. resolve(response);
  819. })
  820. .catch(err => {
  821. reject(err);
  822. });
  823. });
  824. }
  825. /**
  826. * Perform YouTube API get playlist items request
  827. *
  828. * @param {object} payload - object that contains the payload
  829. * @param {object} payload.params - request parameters
  830. * @returns {Promise} - returns promise (reject, resolve)
  831. */
  832. API_GET_PLAYLIST_ITEMS(payload) {
  833. return new Promise((resolve, reject) => {
  834. const { params } = payload;
  835. YouTubeModule.runJob(
  836. "API_CALL",
  837. {
  838. url: "https://www.googleapis.com/youtube/v3/playlistItems",
  839. params: {
  840. key: config.get("apis.youtube.key"),
  841. ...params
  842. },
  843. quotaCost: 1
  844. },
  845. this
  846. )
  847. .then(response => {
  848. resolve(response);
  849. })
  850. .catch(err => {
  851. reject(err);
  852. });
  853. });
  854. }
  855. /**
  856. * Perform YouTube API get channels request
  857. *
  858. * @param {object} payload - object that contains the payload
  859. * @param {object} payload.params - request parameters
  860. * @returns {Promise} - returns promise (reject, resolve)
  861. */
  862. API_GET_CHANNELS(payload) {
  863. return new Promise((resolve, reject) => {
  864. const { params } = payload;
  865. YouTubeModule.runJob(
  866. "API_CALL",
  867. {
  868. url: "https://www.googleapis.com/youtube/v3/channels",
  869. params: {
  870. key: config.get("apis.youtube.key"),
  871. ...params
  872. },
  873. quotaCost: 1
  874. },
  875. this
  876. )
  877. .then(response => {
  878. resolve(response);
  879. })
  880. .catch(err => {
  881. reject(err);
  882. });
  883. });
  884. }
  885. /**
  886. * Perform YouTube API search request
  887. *
  888. * @param {object} payload - object that contains the payload
  889. * @param {object} payload.params - request parameters
  890. * @returns {Promise} - returns promise (reject, resolve)
  891. */
  892. API_SEARCH(payload) {
  893. return new Promise((resolve, reject) => {
  894. const { params } = payload;
  895. YouTubeModule.runJob(
  896. "API_CALL",
  897. {
  898. url: "https://www.googleapis.com/youtube/v3/search",
  899. params: {
  900. key: config.get("apis.youtube.key"),
  901. ...params
  902. },
  903. quotaCost: 100
  904. },
  905. this
  906. )
  907. .then(response => {
  908. resolve(response);
  909. })
  910. .catch(err => {
  911. reject(err);
  912. });
  913. });
  914. }
  915. /**
  916. * Perform YouTube API call
  917. *
  918. * @param {object} payload - object that contains the payload
  919. * @param {object} payload.url - request url
  920. * @param {object} payload.params - request parameters
  921. * @param {object} payload.quotaCost - request quotaCost
  922. * @returns {Promise} - returns promise (reject, resolve)
  923. */
  924. API_CALL(payload) {
  925. return new Promise((resolve, reject) => {
  926. const { url, params, quotaCost } = payload;
  927. const quotaExceeded = isQuotaExceeded(YouTubeModule.apiCalls);
  928. if (quotaExceeded) reject(new Error("Quota has been exceeded. Please wait a while."));
  929. else {
  930. const youtubeApiRequest = new YouTubeModule.YoutubeApiRequestModel({
  931. url,
  932. date: Date.now(),
  933. quotaCost
  934. });
  935. youtubeApiRequest.save();
  936. const { key, ...keylessParams } = payload.params;
  937. CacheModule.runJob(
  938. "HSET",
  939. {
  940. table: "youtubeApiRequestParams",
  941. key: youtubeApiRequest._id.toString(),
  942. value: JSON.stringify(keylessParams)
  943. },
  944. this
  945. ).then();
  946. YouTubeModule.apiCalls.push({ date: youtubeApiRequest.date, quotaCost });
  947. YouTubeModule.axios
  948. .get(url, {
  949. params,
  950. timeout: YouTubeModule.requestTimeout
  951. })
  952. .then(response => {
  953. if (response.data.error) {
  954. reject(new Error(response.data.error));
  955. } else {
  956. CacheModule.runJob(
  957. "HSET",
  958. {
  959. table: "youtubeApiRequestResults",
  960. key: youtubeApiRequest._id.toString(),
  961. value: JSON.stringify(response.data)
  962. },
  963. this
  964. ).then();
  965. resolve({ response });
  966. }
  967. })
  968. .catch(err => {
  969. reject(err);
  970. });
  971. }
  972. });
  973. }
  974. /**
  975. * Fetch all api requests
  976. *
  977. * @param {object} payload - object that contains the payload
  978. * @param {object} payload.fromDate - data to fetch requests up to
  979. * @returns {Promise} - returns promise (reject, resolve)
  980. */
  981. GET_API_REQUESTS(payload) {
  982. return new Promise((resolve, reject) => {
  983. const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
  984. YouTubeModule.youtubeApiRequestModel
  985. .find({ date: { $lte: fromDate } })
  986. .sort({ date: -1 })
  987. .exec((err, youtubeApiRequests) => {
  988. if (err) reject(new Error("Couldn't load YouTube API requests."));
  989. else {
  990. resolve({ apiRequests: youtubeApiRequests });
  991. }
  992. });
  993. });
  994. }
  995. /**
  996. * Fetch an api request
  997. *
  998. * @param {object} payload - object that contains the payload
  999. * @param {object} payload.apiRequestId - the api request id
  1000. * @returns {Promise} - returns promise (reject, resolve)
  1001. */
  1002. GET_API_REQUEST(payload) {
  1003. return new Promise((resolve, reject) => {
  1004. const { apiRequestId } = payload;
  1005. async.waterfall(
  1006. [
  1007. next => {
  1008. YouTubeModule.youtubeApiRequestModel.findOne({ _id: apiRequestId }).exec(next);
  1009. },
  1010. (apiRequest, next) => {
  1011. CacheModule.runJob(
  1012. "HGET",
  1013. {
  1014. table: "youtubeApiRequestParams",
  1015. key: apiRequestId.toString()
  1016. },
  1017. this
  1018. )
  1019. .then(apiRequestParams => {
  1020. next(null, {
  1021. ...apiRequest._doc,
  1022. params: apiRequestParams
  1023. });
  1024. })
  1025. .catch(err => next(err));
  1026. },
  1027. (apiRequest, next) => {
  1028. CacheModule.runJob(
  1029. "HGET",
  1030. {
  1031. table: "youtubeApiRequestResults",
  1032. key: apiRequestId.toString()
  1033. },
  1034. this
  1035. )
  1036. .then(apiRequestResults => {
  1037. next(null, {
  1038. ...apiRequest,
  1039. results: apiRequestResults
  1040. });
  1041. })
  1042. .catch(err => next(err));
  1043. }
  1044. ],
  1045. (err, apiRequest) => {
  1046. if (err) reject(new Error(err));
  1047. else resolve({ apiRequest });
  1048. }
  1049. );
  1050. });
  1051. }
  1052. /**
  1053. * Removed all stored api requests from mongo and redis
  1054. *
  1055. * @returns {Promise} - returns promise (reject, resolve)
  1056. */
  1057. RESET_STORED_API_REQUESTS() {
  1058. return new Promise((resolve, reject) => {
  1059. async.waterfall(
  1060. [
  1061. next => {
  1062. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 1)` });
  1063. YouTubeModule.youtubeApiRequestModel.find({}, next);
  1064. },
  1065. (apiRequests, next) => {
  1066. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 2)` });
  1067. YouTubeModule.youtubeApiRequestModel.deleteMany({}, err => {
  1068. if (err) next("Couldn't reset stored YouTube API requests.");
  1069. else {
  1070. next(null, apiRequests);
  1071. }
  1072. });
  1073. },
  1074. (apiRequests, next) => {
  1075. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 3)` });
  1076. CacheModule.runJob("DEL", { key: "youtubeApiRequestParams" }, this)
  1077. .then(() => next(null, apiRequests))
  1078. .catch(err => next(err));
  1079. },
  1080. (apiRequests, next) => {
  1081. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 4)` });
  1082. CacheModule.runJob("DEL", { key: "youtubeApiRequestResults" }, this)
  1083. .then(() => next(null, apiRequests))
  1084. .catch(err => next(err));
  1085. },
  1086. (apiRequests, next) => {
  1087. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 5)` });
  1088. async.eachLimit(
  1089. apiRequests.map(apiRequest => apiRequest._id),
  1090. 1,
  1091. (requestId, next) => {
  1092. CacheModule.runJob(
  1093. "PUB",
  1094. {
  1095. channel: "youtube.removeYoutubeApiRequest",
  1096. value: requestId
  1097. },
  1098. this
  1099. )
  1100. .then(() => {
  1101. next();
  1102. })
  1103. .catch(err => {
  1104. next(err);
  1105. });
  1106. },
  1107. err => {
  1108. if (err) next(err);
  1109. else next();
  1110. }
  1111. );
  1112. }
  1113. ],
  1114. err => {
  1115. if (err) reject(new Error(err));
  1116. else resolve();
  1117. }
  1118. );
  1119. });
  1120. }
  1121. /**
  1122. * Remove a stored api request
  1123. *
  1124. * @param {object} payload - object that contains the payload
  1125. * @param {object} payload.requestId - the api request id
  1126. * @returns {Promise} - returns promise (reject, resolve)
  1127. */
  1128. REMOVE_STORED_API_REQUEST(payload) {
  1129. return new Promise((resolve, reject) => {
  1130. async.waterfall(
  1131. [
  1132. next => {
  1133. YouTubeModule.youtubeApiRequestModel.deleteOne({ _id: payload.requestId }, err => {
  1134. if (err) next("Couldn't remove stored YouTube API request.");
  1135. else {
  1136. next();
  1137. }
  1138. });
  1139. },
  1140. next => {
  1141. CacheModule.runJob(
  1142. "HDEL",
  1143. {
  1144. table: "youtubeApiRequestParams",
  1145. key: payload.requestId.toString()
  1146. },
  1147. this
  1148. )
  1149. .then(next)
  1150. .catch(err => next(err));
  1151. },
  1152. next => {
  1153. CacheModule.runJob(
  1154. "HDEL",
  1155. {
  1156. table: "youtubeApiRequestResults",
  1157. key: payload.requestId.toString()
  1158. },
  1159. this
  1160. )
  1161. .then(next)
  1162. .catch(err => next(err));
  1163. },
  1164. next => {
  1165. CacheModule.runJob("PUB", {
  1166. channel: "youtube.removeYoutubeApiRequest",
  1167. value: payload.requestId.toString()
  1168. })
  1169. .then(next)
  1170. .catch(err => next(err));
  1171. }
  1172. ],
  1173. err => {
  1174. if (err) reject(new Error(err));
  1175. else resolve();
  1176. }
  1177. );
  1178. });
  1179. }
  1180. /**
  1181. * Create YouTube videos
  1182. *
  1183. * @param {object} payload - an object containing the payload
  1184. * @param {string} payload.youtubeVideos - the youtubeVideo object or array of
  1185. * @returns {Promise} - returns a promise (resolve, reject)
  1186. */
  1187. CREATE_VIDEOS(payload) {
  1188. return new Promise((resolve, reject) => {
  1189. async.waterfall(
  1190. [
  1191. next => {
  1192. let { youtubeVideos } = payload;
  1193. if (typeof youtubeVideos !== "object") next("Invalid youtubeVideos type");
  1194. else {
  1195. if (!Array.isArray(youtubeVideos)) youtubeVideos = [youtubeVideos];
  1196. YouTubeModule.youtubeVideoModel.insertMany(youtubeVideos, next);
  1197. }
  1198. },
  1199. (youtubeVideos, next) => {
  1200. const youtubeIds = youtubeVideos.map(video => video.youtubeId);
  1201. async.eachLimit(
  1202. youtubeIds,
  1203. 2,
  1204. (youtubeId, next) => {
  1205. MediaModule.runJob("RECALCULATE_RATINGS", { youtubeId }, this)
  1206. .then(() => next())
  1207. .catch(next);
  1208. },
  1209. err => {
  1210. if (err) next(err);
  1211. else next(null, youtubeVideos);
  1212. }
  1213. );
  1214. }
  1215. ],
  1216. (err, youtubeVideos) => {
  1217. if (err) reject(new Error(err));
  1218. else resolve({ youtubeVideos });
  1219. }
  1220. );
  1221. });
  1222. }
  1223. /**
  1224. * Get YouTube video
  1225. *
  1226. * @param {object} payload - an object containing the payload
  1227. * @param {string} payload.identifier - the youtube video ObjectId or YouTube ID
  1228. * @param {string} payload.createMissing - attempt to fetch and create video if not in db
  1229. * @returns {Promise} - returns a promise (resolve, reject)
  1230. */
  1231. GET_VIDEO(payload) {
  1232. return new Promise((resolve, reject) => {
  1233. async.waterfall(
  1234. [
  1235. next => {
  1236. const query = mongoose.Types.ObjectId.isValid(payload.identifier)
  1237. ? { _id: payload.identifier }
  1238. : { youtubeId: payload.identifier };
  1239. return YouTubeModule.youtubeVideoModel.findOne(query, next);
  1240. },
  1241. (video, next) => {
  1242. if (video) return next(null, video, false);
  1243. if (mongoose.Types.ObjectId.isValid(payload.identifier) || !payload.createMissing)
  1244. return next("YouTube video not found.");
  1245. const params = {
  1246. part: "snippet,contentDetails,statistics,status",
  1247. id: payload.identifier
  1248. };
  1249. return YouTubeModule.runJob("API_GET_VIDEOS", { params }, this)
  1250. .then(({ response }) => {
  1251. const { data } = response;
  1252. if (data.items[0] === undefined)
  1253. return next("The specified video does not exist or cannot be publicly accessed.");
  1254. // TODO Clean up duration converter
  1255. let dur = data.items[0].contentDetails.duration;
  1256. dur = dur.replace("PT", "");
  1257. let duration = 0;
  1258. dur = dur.replace(/([\d]*)H/, (v, v2) => {
  1259. v2 = Number(v2);
  1260. duration = v2 * 60 * 60;
  1261. return "";
  1262. });
  1263. dur = dur.replace(/([\d]*)M/, (v, v2) => {
  1264. v2 = Number(v2);
  1265. duration += v2 * 60;
  1266. return "";
  1267. });
  1268. dur.replace(/([\d]*)S/, (v, v2) => {
  1269. v2 = Number(v2);
  1270. duration += v2;
  1271. return "";
  1272. });
  1273. const youtubeVideo = {
  1274. youtubeId: data.items[0].id,
  1275. title: data.items[0].snippet.title,
  1276. author: data.items[0].snippet.channelTitle,
  1277. thumbnail: data.items[0].snippet.thumbnails.default.url,
  1278. duration
  1279. };
  1280. return next(null, false, youtubeVideo);
  1281. })
  1282. .catch(next);
  1283. },
  1284. (video, youtubeVideo, next) => {
  1285. if (video) return next(null, video, true);
  1286. return YouTubeModule.runJob("CREATE_VIDEOS", { youtubeVideos: youtubeVideo }, this)
  1287. .then(res => {
  1288. if (res.youtubeVideos.length === 1) next(null, res.youtubeVideos[0], false);
  1289. else next("YouTube video not found.");
  1290. })
  1291. .catch(next);
  1292. }
  1293. ],
  1294. (err, video, existing) => {
  1295. if (err) reject(new Error(err));
  1296. else resolve({ video, existing });
  1297. }
  1298. );
  1299. });
  1300. }
  1301. /**
  1302. * Remove YouTube videos
  1303. *
  1304. * @param {object} payload - an object containing the payload
  1305. * @param {string} payload.videoIds - Array of youtubeVideo ObjectIds
  1306. * @returns {Promise} - returns a promise (resolve, reject)
  1307. */
  1308. REMOVE_VIDEOS(payload) {
  1309. return new Promise((resolve, reject) => {
  1310. let { videoIds } = payload;
  1311. if (!Array.isArray(videoIds)) videoIds = [videoIds];
  1312. async.waterfall(
  1313. [
  1314. next => {
  1315. if (!videoIds.every(videoId => mongoose.Types.ObjectId.isValid(videoId)))
  1316. next("One or more videoIds are not a valid ObjectId.");
  1317. else {
  1318. this.publishProgress({ status: "update", message: `Removing video (stage 1)` });
  1319. YouTubeModule.youtubeVideoModel.find({ _id: { $in: videoIds } }, (err, videos) => {
  1320. if (err) next(err);
  1321. else
  1322. next(
  1323. null,
  1324. videos.map(video => video.youtubeId)
  1325. );
  1326. });
  1327. }
  1328. },
  1329. (youtubeIds, next) => {
  1330. this.publishProgress({ status: "update", message: `Removing video (stage 2)` });
  1331. SongsModule.SongModel.find({ youtubeId: { $in: youtubeIds } }, (err, songs) => {
  1332. if (err) next(err);
  1333. else {
  1334. const filteredIds = youtubeIds.filter(
  1335. youtubeId => !songs.find(song => song.youtubeId === youtubeId)
  1336. );
  1337. if (filteredIds.length < youtubeIds.length)
  1338. next("One or more videos are attached to songs.");
  1339. else next(null, filteredIds);
  1340. }
  1341. });
  1342. },
  1343. (youtubeIds, next) => {
  1344. this.publishProgress({ status: "update", message: `Removing video (stage 3)` });
  1345. MediaModule.runJob("REMOVE_RATINGS", { youtubeIds }, this)
  1346. .then(() => next(null, youtubeIds))
  1347. .catch(next);
  1348. },
  1349. (youtubeIds, next) => {
  1350. this.publishProgress({ status: "update", message: `Removing video (stage 4)` });
  1351. async.eachLimit(
  1352. youtubeIds,
  1353. 2,
  1354. (youtubeId, next) => {
  1355. async.waterfall(
  1356. [
  1357. next => {
  1358. PlaylistsModule.playlistModel.find(
  1359. { "songs.youtubeId": youtubeId },
  1360. (err, playlists) => {
  1361. if (err) next(err);
  1362. else {
  1363. async.eachLimit(
  1364. playlists,
  1365. 1,
  1366. (playlist, next) => {
  1367. PlaylistsModule.runJob(
  1368. "REMOVE_FROM_PLAYLIST",
  1369. { playlistId: playlist._id, youtubeId },
  1370. this
  1371. )
  1372. .then(() => next())
  1373. .catch(next);
  1374. },
  1375. next
  1376. );
  1377. }
  1378. }
  1379. );
  1380. },
  1381. next => {
  1382. StationsModule.stationModel.find(
  1383. { "queue.youtubeId": youtubeId },
  1384. (err, stations) => {
  1385. if (err) next(err);
  1386. else {
  1387. async.eachLimit(
  1388. stations,
  1389. 1,
  1390. (station, next) => {
  1391. StationsModule.runJob(
  1392. "REMOVE_FROM_QUEUE",
  1393. { stationId: station._id, youtubeId },
  1394. this
  1395. )
  1396. .then(() => next())
  1397. .catch(err => {
  1398. if (
  1399. err === "Station not found" ||
  1400. err ===
  1401. "Song is not currently in the queue."
  1402. )
  1403. next();
  1404. else next(err);
  1405. });
  1406. },
  1407. next
  1408. );
  1409. }
  1410. }
  1411. );
  1412. },
  1413. next => {
  1414. StationsModule.stationModel.find(
  1415. { "currentSong.youtubeId": youtubeId },
  1416. (err, stations) => {
  1417. if (err) next(err);
  1418. else {
  1419. async.eachLimit(
  1420. stations,
  1421. 1,
  1422. (station, next) => {
  1423. StationsModule.runJob(
  1424. "SKIP_STATION",
  1425. { stationId: station._id, natural: false },
  1426. this
  1427. )
  1428. .then(() => {
  1429. next();
  1430. })
  1431. .catch(err => {
  1432. if (err.message === "Station not found.")
  1433. next();
  1434. else next(err);
  1435. });
  1436. },
  1437. next
  1438. );
  1439. }
  1440. }
  1441. );
  1442. }
  1443. ],
  1444. next
  1445. );
  1446. },
  1447. next
  1448. );
  1449. },
  1450. next => {
  1451. this.publishProgress({ status: "update", message: `Removing video (stage 5)` });
  1452. YouTubeModule.youtubeVideoModel.deleteMany({ _id: { $in: videoIds } }, next);
  1453. },
  1454. (res, next) => {
  1455. this.publishProgress({ status: "update", message: `Removing video (stage 6)` });
  1456. CacheModule.runJob("PUB", {
  1457. channel: "youtube.removeVideos",
  1458. value: videoIds
  1459. })
  1460. .then(next)
  1461. .catch(err => next(err));
  1462. }
  1463. ],
  1464. err => {
  1465. if (err) reject(new Error(err));
  1466. else resolve();
  1467. }
  1468. );
  1469. });
  1470. }
  1471. /**
  1472. * Request a set of YouTube videos
  1473. *
  1474. * @param {object} payload - an object containing the payload
  1475. * @param {string} payload.url - the url of the the YouTube playlist or channel
  1476. * @param {boolean} payload.musicOnly - whether to only get music from the playlist/channel
  1477. * @param {boolean} payload.returnVideos - whether to return videos
  1478. * @returns {Promise} - returns a promise (resolve, reject)
  1479. */
  1480. REQUEST_SET(payload) {
  1481. return new Promise((resolve, reject) => {
  1482. async.waterfall(
  1483. [
  1484. next => {
  1485. const playlistRegex = /[\\?&]list=([^&#]*)/;
  1486. const channelRegex =
  1487. /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/;
  1488. if (playlistRegex.exec(payload.url) || channelRegex.exec(payload.url))
  1489. YouTubeModule.runJob(
  1490. playlistRegex.exec(payload.url) ? "GET_PLAYLIST" : "GET_CHANNEL",
  1491. {
  1492. url: payload.url,
  1493. musicOnly: payload.musicOnly
  1494. },
  1495. this
  1496. )
  1497. .then(res => {
  1498. next(null, res.songs);
  1499. })
  1500. .catch(next);
  1501. else next("Invalid YouTube URL.");
  1502. },
  1503. (youtubeIds, next) => {
  1504. let successful = 0;
  1505. let failed = 0;
  1506. let alreadyInDatabase = 0;
  1507. let videos = {};
  1508. const successfulVideoIds = [];
  1509. const failedVideoIds = [];
  1510. if (youtubeIds.length === 0) next();
  1511. async.eachOfLimit(
  1512. youtubeIds,
  1513. 1,
  1514. (youtubeId, index, next2) => {
  1515. YouTubeModule.runJob("GET_VIDEO", { identifier: youtubeId, createMissing: true }, this)
  1516. .then(res => {
  1517. successful += 1;
  1518. successfulVideoIds.push(youtubeId);
  1519. if (res.existing) alreadyInDatabase += 1;
  1520. if (res.video) videos[index] = res.video;
  1521. })
  1522. .catch(() => {
  1523. failed += 1;
  1524. failedVideoIds.push(youtubeId);
  1525. })
  1526. .finally(() => {
  1527. next2();
  1528. });
  1529. },
  1530. () => {
  1531. if (payload.returnVideos)
  1532. videos = Object.keys(videos)
  1533. .sort()
  1534. .map(key => videos[key]);
  1535. next(null, {
  1536. successful,
  1537. failed,
  1538. alreadyInDatabase,
  1539. videos,
  1540. successfulVideoIds,
  1541. failedVideoIds
  1542. });
  1543. }
  1544. );
  1545. }
  1546. ],
  1547. (err, response) => {
  1548. if (err) reject(new Error(err));
  1549. else resolve(response);
  1550. }
  1551. );
  1552. });
  1553. }
  1554. }
  1555. export default new _YouTubeModule();