youtube.js 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743
  1. import mongoose from "mongoose";
  2. import async from "async";
  3. import config from "config";
  4. import * as rax from "retry-axios";
  5. import axios from "axios";
  6. import CoreClass from "../core";
  7. class RateLimitter {
  8. /**
  9. * Constructor
  10. *
  11. * @param {number} timeBetween - The time between each allowed YouTube request
  12. */
  13. constructor(timeBetween) {
  14. this.dateStarted = Date.now();
  15. this.timeBetween = timeBetween;
  16. }
  17. /**
  18. * Returns a promise that resolves whenever the ratelimit of a YouTube request is done
  19. *
  20. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  21. */
  22. continue() {
  23. return new Promise(resolve => {
  24. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  25. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  26. });
  27. }
  28. /**
  29. * Restart the rate limit timer
  30. */
  31. restart() {
  32. this.dateStarted = Date.now();
  33. }
  34. }
  35. let YouTubeModule;
  36. let CacheModule;
  37. let DBModule;
  38. let MediaModule;
  39. let SongsModule;
  40. let StationsModule;
  41. let PlaylistsModule;
  42. let WSModule;
  43. const isQuotaExceeded = apiCalls => {
  44. const reversedApiCalls = apiCalls.slice().reverse();
  45. const quotas = config.get("apis.youtube.quotas").slice();
  46. const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
  47. let quotaExceeded = false;
  48. sortedQuotas.forEach(quota => {
  49. let quotaUsed = 0;
  50. let dateCutoff = null;
  51. if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date() - 1000 * 60;
  52. else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date() - 1000 * 100;
  53. else if (quota.type === "QUERIES_PER_DAY") {
  54. // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
  55. dateCutoff = new Date();
  56. dateCutoff.setUTCMilliseconds(0);
  57. dateCutoff.setUTCSeconds(0);
  58. dateCutoff.setUTCMinutes(0);
  59. dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
  60. dateCutoff.setUTCHours(0);
  61. }
  62. reversedApiCalls.forEach(apiCall => {
  63. if (apiCall.date >= dateCutoff) quotaUsed += apiCall.quotaCost;
  64. });
  65. if (quotaUsed >= quota.limit) {
  66. quotaExceeded = true;
  67. }
  68. });
  69. return quotaExceeded;
  70. };
  71. class _YouTubeModule extends CoreClass {
  72. // eslint-disable-next-line require-jsdoc
  73. constructor() {
  74. super("youtube", {
  75. concurrency: 10,
  76. priorities: {
  77. GET_PLAYLIST: 11
  78. }
  79. });
  80. YouTubeModule = this;
  81. }
  82. /**
  83. * Initialises the activities module
  84. *
  85. * @returns {Promise} - returns promise (reject, resolve)
  86. */
  87. async initialize() {
  88. CacheModule = this.moduleManager.modules.cache;
  89. DBModule = this.moduleManager.modules.db;
  90. MediaModule = this.moduleManager.modules.media;
  91. SongsModule = this.moduleManager.modules.songs;
  92. StationsModule = this.moduleManager.modules.stations;
  93. PlaylistsModule = this.moduleManager.modules.playlists;
  94. WSModule = this.moduleManager.modules.ws;
  95. this.youtubeApiRequestModel = this.YoutubeApiRequestModel = await DBModule.runJob("GET_MODEL", {
  96. modelName: "youtubeApiRequest"
  97. });
  98. this.youtubeVideoModel = this.YoutubeVideoModel = await DBModule.runJob("GET_MODEL", {
  99. modelName: "youtubeVideo"
  100. });
  101. return new Promise(resolve => {
  102. CacheModule.runJob("SUB", {
  103. channel: "youtube.removeYoutubeApiRequest",
  104. cb: requestId => {
  105. WSModule.runJob("EMIT_TO_ROOM", {
  106. room: `view-api-request.${requestId}`,
  107. args: ["event:youtubeApiRequest.removed"]
  108. });
  109. WSModule.runJob("EMIT_TO_ROOM", {
  110. room: "admin.youtube",
  111. args: ["event:admin.youtubeApiRequest.removed", { data: { requestId } }]
  112. });
  113. }
  114. });
  115. CacheModule.runJob("SUB", {
  116. channel: "youtube.removeVideos",
  117. cb: videoIds => {
  118. const videos = Array.isArray(videoIds) ? videoIds : [videoIds];
  119. videos.forEach(videoId => {
  120. WSModule.runJob("EMIT_TO_ROOM", {
  121. room: `view-youtube-video.${videoId}`,
  122. args: ["event:youtubeVideo.removed"]
  123. });
  124. WSModule.runJob("EMIT_TO_ROOM", {
  125. room: "admin.youtubeVideos",
  126. args: ["event:admin.youtubeVideo.removed", { data: { videoId } }]
  127. });
  128. WSModule.runJob("EMIT_TO_ROOMS", {
  129. rooms: ["import-album", "edit-songs"],
  130. args: ["event:admin.youtubeVideo.removed", { videoId }]
  131. });
  132. });
  133. }
  134. });
  135. this.rateLimiter = new RateLimitter(config.get("apis.youtube.rateLimit"));
  136. this.requestTimeout = config.get("apis.youtube.requestTimeout");
  137. this.axios = axios.create();
  138. this.axios.defaults.raxConfig = {
  139. instance: this.axios,
  140. retry: config.get("apis.youtube.retryAmount"),
  141. noResponseRetries: config.get("apis.youtube.retryAmount")
  142. };
  143. rax.attach(this.axios);
  144. this.youtubeApiRequestModel
  145. .find(
  146. { date: { $gte: new Date() - 2 * 24 * 60 * 60 * 1000 } },
  147. { date: true, quotaCost: true, _id: false }
  148. )
  149. .sort({ date: 1 })
  150. .exec((err, youtubeApiRequests) => {
  151. if (err) console.log("Couldn't load YouTube API requests.");
  152. else {
  153. this.apiCalls = youtubeApiRequests;
  154. resolve();
  155. }
  156. });
  157. });
  158. }
  159. /**
  160. * Fetches a list of songs from Youtube's API
  161. *
  162. * @param {object} payload - object that contains the payload
  163. * @param {string} payload.query - the query we'll pass to youtubes api
  164. * @param {string} payload.pageToken - (optional) if this exists, will search search youtube for a specific page reference
  165. * @returns {Promise} - returns promise (reject, resolve)
  166. */
  167. SEARCH(payload) {
  168. const params = {
  169. part: "snippet",
  170. q: payload.query,
  171. type: "video",
  172. maxResults: 10
  173. };
  174. if (payload.pageToken) params.pageToken = payload.pageToken;
  175. return new Promise((resolve, reject) => {
  176. YouTubeModule.runJob(
  177. "API_SEARCH",
  178. {
  179. params
  180. },
  181. this
  182. )
  183. .then(({ response }) => {
  184. const { data } = response;
  185. return resolve(data);
  186. })
  187. .catch(err => {
  188. YouTubeModule.log("ERROR", "SEARCH", `${err.message}`);
  189. if (err.message === "Searching with YouTube is disabled.") return reject(err);
  190. return reject(new Error("An error has occured. Please try again later."));
  191. });
  192. });
  193. }
  194. /**
  195. * Returns details about the YouTube quota usage
  196. *
  197. * @param {object} payload - object that contains the payload
  198. * @param {string} payload.fromDate - date to select requests up to
  199. * @returns {Promise} - returns promise (reject, resolve)
  200. */
  201. GET_QUOTA_STATUS(payload) {
  202. return new Promise((resolve, reject) => {
  203. const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
  204. YouTubeModule.youtubeApiRequestModel
  205. .find(
  206. { date: { $gte: fromDate - 2 * 24 * 60 * 60 * 1000, $lte: fromDate } },
  207. { date: true, quotaCost: true, _id: false }
  208. )
  209. .sort({ date: 1 })
  210. .exec((err, youtubeApiRequests) => {
  211. if (err) reject(new Error("Couldn't load YouTube API requests."));
  212. else {
  213. const reversedApiCalls = youtubeApiRequests.slice().reverse();
  214. const quotas = config.get("apis.youtube.quotas").slice();
  215. const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
  216. const status = {};
  217. sortedQuotas.forEach(quota => {
  218. status[quota.type] = {
  219. title: quota.title,
  220. quotaUsed: 0,
  221. limit: quota.limit,
  222. quotaExceeded: false
  223. };
  224. let dateCutoff = null;
  225. if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date(fromDate) - 1000 * 60;
  226. else if (quota.type === "QUERIES_PER_100_SECONDS")
  227. dateCutoff = new Date(fromDate) - 1000 * 100;
  228. else if (quota.type === "QUERIES_PER_DAY") {
  229. // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
  230. dateCutoff = new Date(fromDate);
  231. dateCutoff.setUTCMilliseconds(0);
  232. dateCutoff.setUTCSeconds(0);
  233. dateCutoff.setUTCMinutes(0);
  234. dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
  235. dateCutoff.setUTCHours(0);
  236. }
  237. reversedApiCalls.forEach(apiCall => {
  238. if (apiCall.date >= dateCutoff) status[quota.type].quotaUsed += apiCall.quotaCost;
  239. });
  240. if (status[quota.type].quotaUsed >= quota.limit && !status[quota.type].quotaExceeded)
  241. status[quota.type].quotaExceeded = true;
  242. });
  243. resolve({ status });
  244. }
  245. });
  246. });
  247. }
  248. /**
  249. * Returns YouTube quota chart data
  250. *
  251. * @param {object} payload - object that contains the payload
  252. * @param {string} payload.timePeriod - either hours or days
  253. * @param {string} payload.startDate - beginning date
  254. * @param {string} payload.endDate - end date
  255. * @param {string} payload.dataType - either usage or count
  256. * @returns {Promise} - returns promise (reject, resolve)
  257. */
  258. GET_QUOTA_CHART_DATA(payload) {
  259. return new Promise((resolve, reject) => {
  260. const { timePeriod, startDate, endDate, dataType } = payload;
  261. // const timePeriod = "hours";
  262. // const startDate = new Date("2022-05-20 00:00:00");
  263. // const endDate = new Date("2022-05-21 00:00:00");
  264. // const timePeriod = "days";
  265. // const startDate = new Date("2022-05-15 00:00:00");
  266. // const endDate = new Date("2022-05-21 00:00:00");
  267. // const endDate = new Date("2022-05-30 00:00:00");
  268. // const dataType = "usage";
  269. // const dataType = "count";
  270. async.waterfall(
  271. [
  272. next => {
  273. let timeRanges = [];
  274. if (timePeriod === "hours") {
  275. startDate.setMinutes(0, 0, 0);
  276. endDate.setMinutes(0, 0, 0);
  277. const lastDate = new Date(startDate);
  278. do {
  279. const newDate = new Date(lastDate.getTime() + 1000 * 60 * 60);
  280. timeRanges.push({
  281. startDate: new Date(lastDate),
  282. endDate: newDate
  283. });
  284. lastDate.setTime(newDate.getTime());
  285. } while (lastDate.getTime() < endDate.getTime());
  286. if (timeRanges.length === 0 || timeRanges.length > 24)
  287. return next("No valid time ranges specified.");
  288. timeRanges = timeRanges.map(timeRange => ({
  289. ...timeRange,
  290. label: `${timeRange.startDate.getHours().toString().padStart(2, "0")}:00`
  291. }));
  292. } else if (timePeriod === "days") {
  293. startDate.setHours(0, 0, 0, 0);
  294. endDate.setHours(0, 0, 0, 0);
  295. const lastDate = new Date(startDate);
  296. do {
  297. const newDate = new Date(lastDate.getTime() + 1000 * 60 * 60 * 24);
  298. timeRanges.push({
  299. startDate: new Date(lastDate),
  300. endDate: newDate
  301. });
  302. lastDate.setTime(newDate.getTime());
  303. } while (lastDate.getTime() < endDate.getTime());
  304. if (timeRanges.length === 0 || timeRanges.length > 31)
  305. return next("No valid time ranges specified.");
  306. const days = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
  307. if (timeRanges.length <= 7)
  308. timeRanges = timeRanges.map(timeRange => ({
  309. ...timeRange,
  310. label: days[timeRange.startDate.getDay()]
  311. }));
  312. else
  313. timeRanges = timeRanges.map(timeRange => ({
  314. ...timeRange,
  315. label: `${timeRange.startDate.getDate().toString().padStart(2, "0")}-${(
  316. timeRange.startDate.getMonth() + 1
  317. )
  318. .toString()
  319. .padStart(2, "0")}`
  320. }));
  321. }
  322. return next(null, timeRanges);
  323. },
  324. (timeRanges, next) => {
  325. YouTubeModule.youtubeApiRequestModel
  326. .find({
  327. date: { $gte: startDate, $lte: endDate }
  328. })
  329. .sort({ date: 1 })
  330. .exec((err, youtubeApiRequests) => {
  331. next(err, timeRanges, youtubeApiRequests);
  332. });
  333. },
  334. (timeRanges, youtubeApiRequests, next) => {
  335. const regex = /https:\/\/www\.googleapis\.com\/youtube\/v3\/(.+)/;
  336. const requestTypes = Object.fromEntries(
  337. youtubeApiRequests
  338. .map(youtubeApiRequest => regex.exec(youtubeApiRequest.url)[1])
  339. .filter((requestType, index, array) => array.indexOf(requestType) === index)
  340. .map(requestType => [requestType, 0])
  341. );
  342. timeRanges = timeRanges.map(timeRange => ({
  343. ...timeRange,
  344. data: { total: 0, ...requestTypes }
  345. }));
  346. youtubeApiRequests.forEach(youtubeApiRequest => {
  347. timeRanges.forEach(timeRange => {
  348. if (
  349. timeRange.startDate <= youtubeApiRequest.date &&
  350. timeRange.endDate >= youtubeApiRequest.date
  351. ) {
  352. const requestType = regex.exec(youtubeApiRequest.url)[1];
  353. if (!timeRange.data[requestType]) timeRange.data[requestType] = 0;
  354. if (dataType === "usage") {
  355. timeRange.data[requestType] += youtubeApiRequest.quotaCost;
  356. timeRange.data.total += youtubeApiRequest.quotaCost;
  357. } else if (dataType === "count") {
  358. timeRange.data[requestType] += 1;
  359. timeRange.data.total += 1;
  360. }
  361. }
  362. });
  363. });
  364. next(null, timeRanges);
  365. },
  366. (timeRanges, next) => {
  367. const chartData = {};
  368. chartData.labels = timeRanges.map(timeRange => timeRange.label);
  369. const datasetTypes = Object.keys(timeRanges[0].data);
  370. const colors = {
  371. total: "rgb(2, 166, 242)",
  372. videos: "rgb(166, 2, 242)",
  373. search: "rgb(242, 2, 166)",
  374. channels: "rgb(2, 242, 166)",
  375. playlistItems: "rgb(242, 166, 2)"
  376. };
  377. chartData.datasets = datasetTypes.map(datasetType => {
  378. let label;
  379. switch (datasetType) {
  380. case "total":
  381. label = "Total";
  382. break;
  383. case "videos":
  384. label = "Videos";
  385. break;
  386. case "search":
  387. label = "Search";
  388. break;
  389. case "playlistItems":
  390. label = "Playlist Items";
  391. break;
  392. default:
  393. label = datasetType;
  394. }
  395. return {
  396. label,
  397. borderColor: colors[datasetType],
  398. backgroundColor: colors[datasetType],
  399. data: timeRanges.map(timeRange => timeRange.data[datasetType])
  400. };
  401. });
  402. next(null, chartData);
  403. }
  404. ],
  405. (err, chartData) => {
  406. if (err) reject(err);
  407. else resolve(chartData);
  408. }
  409. );
  410. });
  411. }
  412. /**
  413. * Gets the id of the channel upload playlist
  414. *
  415. * @param {object} payload - object that contains the payload
  416. * @param {string} payload.id - the id of the YouTube channel. Optional: can be left out if specifying a username.
  417. * @param {string} payload.username - the username of the YouTube channel. Only gets used if no id is specified.
  418. * @returns {Promise} - returns promise (reject, resolve)
  419. */
  420. GET_CHANNEL_UPLOADS_PLAYLIST_ID(payload) {
  421. return new Promise((resolve, reject) => {
  422. const params = {
  423. part: "id,contentDetails"
  424. };
  425. if (payload.id) params.id = payload.id;
  426. else params.forUsername = payload.username;
  427. YouTubeModule.runJob(
  428. "API_GET_CHANNELS",
  429. {
  430. params
  431. },
  432. this
  433. )
  434. .then(({ response }) => {
  435. const { data } = response;
  436. if (data.pageInfo.totalResults === 0) return reject(new Error("Channel not found."));
  437. const playlistId = data.items[0].contentDetails.relatedPlaylists.uploads;
  438. return resolve({ playlistId });
  439. })
  440. .catch(err => {
  441. YouTubeModule.log("ERROR", "GET_CHANNEL_UPLOADS_PLAYLIST_ID", `${err.message}`);
  442. if (err.message === "Request failed with status code 404")
  443. return reject(new Error("Channel not found. Is the channel public/unlisted?"));
  444. if (err.message === "Searching with YouTube is disabled.") return reject(err);
  445. return reject(new Error("An error has occured. Please try again later."));
  446. });
  447. });
  448. }
  449. /**
  450. * Gets the id of the channel from the custom URL
  451. *
  452. * @param {object} payload - object that contains the payload
  453. * @param {string} payload.customUrl - the customUrl of the YouTube channel
  454. * @returns {Promise} - returns promise (reject, resolve)
  455. */
  456. GET_CHANNEL_ID_FROM_CUSTOM_URL(payload) {
  457. return new Promise((resolve, reject) => {
  458. async.waterfall(
  459. [
  460. next => {
  461. const params = {
  462. part: "snippet",
  463. type: "channel",
  464. maxResults: 50
  465. };
  466. params.q = payload.customUrl;
  467. YouTubeModule.runJob(
  468. "API_SEARCH",
  469. {
  470. params
  471. },
  472. this
  473. )
  474. .then(({ response }) => {
  475. const { data } = response;
  476. if (data.pageInfo.totalResults === 0) return next("Channel not found.");
  477. const channelIds = data.items.map(item => item.id.channelId);
  478. return next(null, channelIds);
  479. })
  480. .catch(err => {
  481. next(err);
  482. });
  483. },
  484. (channelIds, next) => {
  485. const params = {
  486. part: "snippet",
  487. id: channelIds.join(","),
  488. maxResults: 50
  489. };
  490. YouTubeModule.runJob(
  491. "API_GET_CHANNELS",
  492. {
  493. params
  494. },
  495. this
  496. )
  497. .then(({ response }) => {
  498. const { data } = response;
  499. if (data.pageInfo.totalResults === 0) return next("Channel not found.");
  500. let channelId = null;
  501. data.items.forEach(item => {
  502. if (
  503. item.snippet.customUrl &&
  504. item.snippet.customUrl.toLowerCase() === payload.customUrl.toLowerCase()
  505. ) {
  506. channelId = item.id;
  507. }
  508. });
  509. if (!channelId) return next("Channel not found.");
  510. return next(null, channelId);
  511. })
  512. .catch(err => {
  513. next(err);
  514. });
  515. }
  516. ],
  517. (err, channelId) => {
  518. if (err) {
  519. YouTubeModule.log("ERROR", "GET_CHANNEL_ID_FROM_CUSTOM_URL", `${err.message || err}`);
  520. if (err.message === "Request failed with status code 404")
  521. return reject(new Error("Channel not found. Is the channel public/unlisted?"));
  522. if (err.message === "Searching with YouTube is disabled.") return reject(err);
  523. return reject(new Error("An error has occured. Please try again later."));
  524. }
  525. return resolve({ channelId });
  526. }
  527. );
  528. });
  529. }
  530. /**
  531. * Returns an array of songs taken from a YouTube playlist
  532. *
  533. * @param {object} payload - object that contains the payload
  534. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the playlist
  535. * @param {string} payload.url - the url of the YouTube playlist
  536. * @returns {Promise} - returns promise (reject, resolve)
  537. */
  538. GET_PLAYLIST(payload) {
  539. return new Promise((resolve, reject) => {
  540. const regex = /[\\?&]list=([^&#]*)/;
  541. const splitQuery = regex.exec(payload.url);
  542. if (!splitQuery) {
  543. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Invalid YouTube playlist URL query.");
  544. reject(new Error("Invalid playlist URL."));
  545. return;
  546. }
  547. const playlistId = splitQuery[1];
  548. async.waterfall(
  549. [
  550. next => {
  551. let songs = [];
  552. let nextPageToken = "";
  553. async.whilst(
  554. next => {
  555. YouTubeModule.log(
  556. "INFO",
  557. `Getting playlist progress for job (${this.toString()}): ${
  558. songs.length
  559. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  560. );
  561. next(null, nextPageToken !== undefined);
  562. },
  563. next => {
  564. // Add 250ms delay between each job request
  565. setTimeout(() => {
  566. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  567. .then(response => {
  568. songs = songs.concat(response.songs);
  569. nextPageToken = response.nextPageToken;
  570. next();
  571. })
  572. .catch(err => next(err));
  573. }, 250);
  574. },
  575. err => next(err, songs)
  576. );
  577. },
  578. (songs, next) =>
  579. next(
  580. null,
  581. songs.map(song => song.contentDetails.videoId)
  582. ),
  583. (songs, next) => {
  584. if (!payload.musicOnly) return next(true, { songs });
  585. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  586. .then(filteredSongs => next(null, { filteredSongs, songs }))
  587. .catch(next);
  588. }
  589. ],
  590. (err, response) => {
  591. if (err && err !== true) {
  592. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Some error has occurred.", err.message);
  593. reject(new Error(err.message));
  594. } else {
  595. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  596. }
  597. }
  598. );
  599. });
  600. }
  601. /**
  602. * Returns a a page from a YouTube playlist. Is used internally by GET_PLAYLIST and GET_CHANNEL.
  603. *
  604. * @param {object} payload - object that contains the payload
  605. * @param {boolean} payload.playlistId - the playlist id to get videos from
  606. * @param {boolean} payload.nextPageToken - the nextPageToken to use
  607. * @param {string} payload.url - the url of the YouTube playlist
  608. * @returns {Promise} - returns promise (reject, resolve)
  609. */
  610. GET_PLAYLIST_PAGE(payload) {
  611. return new Promise((resolve, reject) => {
  612. const params = {
  613. part: "contentDetails",
  614. playlistId: payload.playlistId,
  615. maxResults: 50
  616. };
  617. if (payload.nextPageToken) params.pageToken = payload.nextPageToken;
  618. YouTubeModule.runJob(
  619. "API_GET_PLAYLIST_ITEMS",
  620. {
  621. params
  622. },
  623. this
  624. )
  625. .then(({ response }) => {
  626. const { data } = response;
  627. const songs = data.items;
  628. if (data.nextPageToken) return resolve({ nextPageToken: data.nextPageToken, songs });
  629. return resolve({ songs });
  630. })
  631. .catch(err => {
  632. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${err.message}`);
  633. if (err.message === "Request failed with status code 404")
  634. return reject(new Error("Playlist not found. Is the playlist public/unlisted?"));
  635. return reject(new Error("An error has occured. Please try again later."));
  636. });
  637. });
  638. }
  639. /**
  640. * Filters a list of YouTube videos so that they only contains videos with music. Is used internally by GET_PLAYLIST
  641. *
  642. * @param {object} payload - object that contains the payload
  643. * @param {Array} payload.videoIds - an array of YouTube videoIds to filter through
  644. * @param {Array} payload.page - the current page/set of video's to get, starting at 0. If left null, 0 is assumed. Will recurse.
  645. * @returns {Promise} - returns promise (reject, resolve)
  646. */
  647. FILTER_MUSIC_VIDEOS(payload) {
  648. return new Promise((resolve, reject) => {
  649. const page = payload.page ? payload.page : 0;
  650. const videosPerPage = 50;
  651. const localVideoIds = payload.videoIds.splice(page * 50, videosPerPage);
  652. if (localVideoIds.length === 0) {
  653. resolve({ videoIds: [] });
  654. return;
  655. }
  656. const params = {
  657. part: "topicDetails",
  658. id: localVideoIds.join(","),
  659. maxResults: videosPerPage
  660. };
  661. YouTubeModule.runJob("API_GET_VIDEOS", { params }, this)
  662. .then(({ response }) => {
  663. const { data } = response;
  664. const videoIds = [];
  665. data.items.forEach(item => {
  666. const videoId = item.id;
  667. if (!item.topicDetails) return;
  668. if (item.topicDetails.topicCategories.indexOf("https://en.wikipedia.org/wiki/Music") !== -1)
  669. videoIds.push(videoId);
  670. });
  671. return YouTubeModule.runJob(
  672. "FILTER_MUSIC_VIDEOS",
  673. { videoIds: payload.videoIds, page: page + 1 },
  674. this
  675. )
  676. .then(result => resolve({ videoIds: videoIds.concat(result.videoIds) }))
  677. .catch(err => reject(err));
  678. })
  679. .catch(err => {
  680. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${err.message}`);
  681. return reject(new Error("Failed to find playlist from YouTube"));
  682. });
  683. });
  684. }
  685. /**
  686. * Returns an array of songs taken from a YouTube channel
  687. *
  688. * @param {object} payload - object that contains the payload
  689. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the channel
  690. * @param {boolean} payload.disableSearch - whether to allow searching for custom url/username
  691. * @param {string} payload.url - the url of the YouTube channel
  692. * @returns {Promise} - returns promise (reject, resolve)
  693. */
  694. GET_CHANNEL(payload) {
  695. return new Promise((resolve, reject) => {
  696. const regex =
  697. /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/;
  698. const splitQuery = regex.exec(payload.url);
  699. if (!splitQuery) {
  700. YouTubeModule.log("ERROR", "GET_CHANNEL", "Invalid YouTube channel URL query.");
  701. reject(new Error("Invalid playlist URL."));
  702. return;
  703. }
  704. const channelId = splitQuery[1];
  705. const channelUsername = splitQuery[2];
  706. const channelCustomUrl = splitQuery[3];
  707. const channelUsernameOrCustomUrl = splitQuery[4];
  708. console.log(`Channel id: ${channelId}`);
  709. console.log(`Channel username: ${channelUsername}`);
  710. console.log(`Channel custom URL: ${channelCustomUrl}`);
  711. console.log(`Channel username or custom URL: ${channelUsernameOrCustomUrl}`);
  712. const disableSearch = payload.disableSearch || false;
  713. async.waterfall(
  714. [
  715. next => {
  716. const payload = {};
  717. if (channelId) payload.id = channelId;
  718. else if (channelUsername) payload.username = channelUsername;
  719. else return next(null, true, null);
  720. return YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", payload, this)
  721. .then(({ playlistId }) => {
  722. next(null, false, playlistId);
  723. })
  724. .catch(err => {
  725. if (err.message === "Channel not found. Is the channel public/unlisted?")
  726. next(null, true, null);
  727. else next(err);
  728. });
  729. },
  730. (getUsernameFromCustomUrl, playlistId, next) => {
  731. if (!getUsernameFromCustomUrl) return next(null, playlistId);
  732. if (disableSearch)
  733. return next(
  734. "Importing with this type of URL is disabled. Please provide a channel URL with the channel ID."
  735. );
  736. const payload = {};
  737. if (channelCustomUrl) payload.customUrl = channelCustomUrl;
  738. else if (channelUsernameOrCustomUrl) payload.customUrl = channelUsernameOrCustomUrl;
  739. else return next("No proper URL provided.");
  740. return YouTubeModule.runJob("GET_CHANNEL_ID_FROM_CUSTOM_URL", payload, this)
  741. .then(({ channelId }) => {
  742. YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", { id: channelId }, this)
  743. .then(({ playlistId }) => {
  744. next(null, playlistId);
  745. })
  746. .catch(err => next(err));
  747. })
  748. .catch(err => next(err));
  749. },
  750. (playlistId, next) => {
  751. let songs = [];
  752. let nextPageToken = "";
  753. async.whilst(
  754. next => {
  755. YouTubeModule.log(
  756. "INFO",
  757. `Getting channel progress for job (${this.toString()}): ${
  758. songs.length
  759. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  760. );
  761. next(null, nextPageToken !== undefined);
  762. },
  763. next => {
  764. // Add 250ms delay between each job request
  765. setTimeout(() => {
  766. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  767. .then(response => {
  768. songs = songs.concat(response.songs);
  769. nextPageToken = response.nextPageToken;
  770. next();
  771. })
  772. .catch(err => next(err));
  773. }, 250);
  774. },
  775. err => next(err, songs)
  776. );
  777. },
  778. (songs, next) =>
  779. next(
  780. null,
  781. songs.map(song => song.contentDetails.videoId)
  782. ),
  783. (songs, next) => {
  784. if (!payload.musicOnly) return next(true, { songs });
  785. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  786. .then(filteredSongs => next(null, { filteredSongs, songs }))
  787. .catch(next);
  788. }
  789. ],
  790. (err, response) => {
  791. if (err && err !== true) {
  792. YouTubeModule.log("ERROR", "GET_CHANNEL", "Some error has occurred.", err.message || err);
  793. reject(new Error(err.message || err));
  794. } else {
  795. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  796. }
  797. }
  798. );
  799. });
  800. }
  801. /**
  802. * Perform YouTube API get videos request
  803. *
  804. * @param {object} payload - object that contains the payload
  805. * @param {object} payload.params - request parameters
  806. * @returns {Promise} - returns promise (reject, resolve)
  807. */
  808. API_GET_VIDEOS(payload) {
  809. return new Promise((resolve, reject) => {
  810. const { params } = payload;
  811. YouTubeModule.runJob(
  812. "API_CALL",
  813. {
  814. url: "https://www.googleapis.com/youtube/v3/videos",
  815. params: {
  816. key: config.get("apis.youtube.key"),
  817. ...params
  818. },
  819. quotaCost: 1
  820. },
  821. this
  822. )
  823. .then(response => {
  824. resolve(response);
  825. })
  826. .catch(err => {
  827. reject(err);
  828. });
  829. });
  830. }
  831. /**
  832. * Perform YouTube API get playlist items request
  833. *
  834. * @param {object} payload - object that contains the payload
  835. * @param {object} payload.params - request parameters
  836. * @returns {Promise} - returns promise (reject, resolve)
  837. */
  838. API_GET_PLAYLIST_ITEMS(payload) {
  839. return new Promise((resolve, reject) => {
  840. const { params } = payload;
  841. YouTubeModule.runJob(
  842. "API_CALL",
  843. {
  844. url: "https://www.googleapis.com/youtube/v3/playlistItems",
  845. params: {
  846. key: config.get("apis.youtube.key"),
  847. ...params
  848. },
  849. quotaCost: 1
  850. },
  851. this
  852. )
  853. .then(response => {
  854. resolve(response);
  855. })
  856. .catch(err => {
  857. reject(err);
  858. });
  859. });
  860. }
  861. /**
  862. * Perform YouTube API get channels request
  863. *
  864. * @param {object} payload - object that contains the payload
  865. * @param {object} payload.params - request parameters
  866. * @returns {Promise} - returns promise (reject, resolve)
  867. */
  868. API_GET_CHANNELS(payload) {
  869. return new Promise((resolve, reject) => {
  870. const { params } = payload;
  871. YouTubeModule.runJob(
  872. "API_CALL",
  873. {
  874. url: "https://www.googleapis.com/youtube/v3/channels",
  875. params: {
  876. key: config.get("apis.youtube.key"),
  877. ...params
  878. },
  879. quotaCost: 1
  880. },
  881. this
  882. )
  883. .then(response => {
  884. resolve(response);
  885. })
  886. .catch(err => {
  887. reject(err);
  888. });
  889. });
  890. }
  891. /**
  892. * Perform YouTube API search request
  893. *
  894. * @param {object} payload - object that contains the payload
  895. * @param {object} payload.params - request parameters
  896. * @returns {Promise} - returns promise (reject, resolve)
  897. */
  898. API_SEARCH(payload) {
  899. return new Promise((resolve, reject) => {
  900. const { params } = payload;
  901. if (
  902. config.has("experimental.disable_youtube_search") &&
  903. config.get("experimental.disable_youtube_search")
  904. ) {
  905. reject(new Error("Searching with YouTube is disabled."));
  906. return;
  907. }
  908. YouTubeModule.runJob(
  909. "API_CALL",
  910. {
  911. url: "https://www.googleapis.com/youtube/v3/search",
  912. params: {
  913. key: config.get("apis.youtube.key"),
  914. ...params
  915. },
  916. quotaCost: 100
  917. },
  918. this
  919. )
  920. .then(response => {
  921. resolve(response);
  922. })
  923. .catch(err => {
  924. reject(err);
  925. });
  926. });
  927. }
  928. /**
  929. * Perform YouTube API call
  930. *
  931. * @param {object} payload - object that contains the payload
  932. * @param {object} payload.url - request url
  933. * @param {object} payload.params - request parameters
  934. * @param {object} payload.quotaCost - request quotaCost
  935. * @returns {Promise} - returns promise (reject, resolve)
  936. */
  937. API_CALL(payload) {
  938. return new Promise((resolve, reject) => {
  939. const { url, params, quotaCost } = payload;
  940. const quotaExceeded = isQuotaExceeded(YouTubeModule.apiCalls);
  941. if (quotaExceeded) reject(new Error("Quota has been exceeded. Please wait a while."));
  942. else {
  943. const youtubeApiRequest = new YouTubeModule.YoutubeApiRequestModel({
  944. url,
  945. date: Date.now(),
  946. quotaCost
  947. });
  948. youtubeApiRequest.save();
  949. const { ...keylessParams } = payload.params;
  950. CacheModule.runJob(
  951. "HSET",
  952. {
  953. table: "youtubeApiRequestParams",
  954. key: youtubeApiRequest._id.toString(),
  955. value: JSON.stringify(keylessParams)
  956. },
  957. this
  958. ).then();
  959. YouTubeModule.apiCalls.push({ date: youtubeApiRequest.date, quotaCost });
  960. YouTubeModule.axios
  961. .get(url, {
  962. params,
  963. timeout: YouTubeModule.requestTimeout
  964. })
  965. .then(response => {
  966. if (response.data.error) {
  967. reject(new Error(response.data.error));
  968. } else {
  969. CacheModule.runJob(
  970. "HSET",
  971. {
  972. table: "youtubeApiRequestResults",
  973. key: youtubeApiRequest._id.toString(),
  974. value: JSON.stringify(response.data)
  975. },
  976. this
  977. ).then();
  978. resolve({ response });
  979. }
  980. })
  981. .catch(err => {
  982. reject(err);
  983. });
  984. }
  985. });
  986. }
  987. /**
  988. * Fetch all api requests
  989. *
  990. * @param {object} payload - object that contains the payload
  991. * @param {object} payload.fromDate - data to fetch requests up to
  992. * @returns {Promise} - returns promise (reject, resolve)
  993. */
  994. GET_API_REQUESTS(payload) {
  995. return new Promise((resolve, reject) => {
  996. const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
  997. YouTubeModule.youtubeApiRequestModel
  998. .find({ date: { $lte: fromDate } })
  999. .sort({ date: -1 })
  1000. .exec((err, youtubeApiRequests) => {
  1001. if (err) reject(new Error("Couldn't load YouTube API requests."));
  1002. else {
  1003. resolve({ apiRequests: youtubeApiRequests });
  1004. }
  1005. });
  1006. });
  1007. }
  1008. /**
  1009. * Fetch an api request
  1010. *
  1011. * @param {object} payload - object that contains the payload
  1012. * @param {object} payload.apiRequestId - the api request id
  1013. * @returns {Promise} - returns promise (reject, resolve)
  1014. */
  1015. GET_API_REQUEST(payload) {
  1016. return new Promise((resolve, reject) => {
  1017. const { apiRequestId } = payload;
  1018. async.waterfall(
  1019. [
  1020. next => {
  1021. YouTubeModule.youtubeApiRequestModel.findOne({ _id: apiRequestId }).exec(next);
  1022. },
  1023. (apiRequest, next) => {
  1024. CacheModule.runJob(
  1025. "HGET",
  1026. {
  1027. table: "youtubeApiRequestParams",
  1028. key: apiRequestId.toString()
  1029. },
  1030. this
  1031. )
  1032. .then(apiRequestParams => {
  1033. next(null, {
  1034. ...apiRequest._doc,
  1035. params: apiRequestParams
  1036. });
  1037. })
  1038. .catch(err => next(err));
  1039. },
  1040. (apiRequest, next) => {
  1041. CacheModule.runJob(
  1042. "HGET",
  1043. {
  1044. table: "youtubeApiRequestResults",
  1045. key: apiRequestId.toString()
  1046. },
  1047. this
  1048. )
  1049. .then(apiRequestResults => {
  1050. next(null, {
  1051. ...apiRequest,
  1052. results: apiRequestResults
  1053. });
  1054. })
  1055. .catch(err => next(err));
  1056. }
  1057. ],
  1058. (err, apiRequest) => {
  1059. if (err) reject(new Error(err));
  1060. else resolve({ apiRequest });
  1061. }
  1062. );
  1063. });
  1064. }
  1065. /**
  1066. * Removed all stored api requests from mongo and redis
  1067. *
  1068. * @returns {Promise} - returns promise (reject, resolve)
  1069. */
  1070. RESET_STORED_API_REQUESTS() {
  1071. return new Promise((resolve, reject) => {
  1072. async.waterfall(
  1073. [
  1074. next => {
  1075. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 1)` });
  1076. YouTubeModule.youtubeApiRequestModel.find({}, next);
  1077. },
  1078. (apiRequests, next) => {
  1079. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 2)` });
  1080. YouTubeModule.youtubeApiRequestModel.deleteMany({}, err => {
  1081. if (err) next("Couldn't reset stored YouTube API requests.");
  1082. else {
  1083. next(null, apiRequests);
  1084. }
  1085. });
  1086. },
  1087. (apiRequests, next) => {
  1088. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 3)` });
  1089. CacheModule.runJob("DEL", { key: "youtubeApiRequestParams" }, this)
  1090. .then(() => next(null, apiRequests))
  1091. .catch(err => next(err));
  1092. },
  1093. (apiRequests, next) => {
  1094. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 4)` });
  1095. CacheModule.runJob("DEL", { key: "youtubeApiRequestResults" }, this)
  1096. .then(() => next(null, apiRequests))
  1097. .catch(err => next(err));
  1098. },
  1099. (apiRequests, next) => {
  1100. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 5)` });
  1101. async.eachLimit(
  1102. apiRequests.map(apiRequest => apiRequest._id),
  1103. 1,
  1104. (requestId, next) => {
  1105. CacheModule.runJob(
  1106. "PUB",
  1107. {
  1108. channel: "youtube.removeYoutubeApiRequest",
  1109. value: requestId
  1110. },
  1111. this
  1112. )
  1113. .then(() => {
  1114. next();
  1115. })
  1116. .catch(err => {
  1117. next(err);
  1118. });
  1119. },
  1120. err => {
  1121. if (err) next(err);
  1122. else next();
  1123. }
  1124. );
  1125. }
  1126. ],
  1127. err => {
  1128. if (err) reject(new Error(err));
  1129. else resolve();
  1130. }
  1131. );
  1132. });
  1133. }
  1134. /**
  1135. * Remove a stored api request
  1136. *
  1137. * @param {object} payload - object that contains the payload
  1138. * @param {object} payload.requestId - the api request id
  1139. * @returns {Promise} - returns promise (reject, resolve)
  1140. */
  1141. REMOVE_STORED_API_REQUEST(payload) {
  1142. return new Promise((resolve, reject) => {
  1143. async.waterfall(
  1144. [
  1145. next => {
  1146. YouTubeModule.youtubeApiRequestModel.deleteOne({ _id: payload.requestId }, err => {
  1147. if (err) next("Couldn't remove stored YouTube API request.");
  1148. else {
  1149. next();
  1150. }
  1151. });
  1152. },
  1153. next => {
  1154. CacheModule.runJob(
  1155. "HDEL",
  1156. {
  1157. table: "youtubeApiRequestParams",
  1158. key: payload.requestId.toString()
  1159. },
  1160. this
  1161. )
  1162. .then(next)
  1163. .catch(err => next(err));
  1164. },
  1165. next => {
  1166. CacheModule.runJob(
  1167. "HDEL",
  1168. {
  1169. table: "youtubeApiRequestResults",
  1170. key: payload.requestId.toString()
  1171. },
  1172. this
  1173. )
  1174. .then(next)
  1175. .catch(err => next(err));
  1176. },
  1177. next => {
  1178. CacheModule.runJob("PUB", {
  1179. channel: "youtube.removeYoutubeApiRequest",
  1180. value: payload.requestId.toString()
  1181. })
  1182. .then(next)
  1183. .catch(err => next(err));
  1184. }
  1185. ],
  1186. err => {
  1187. if (err) reject(new Error(err));
  1188. else resolve();
  1189. }
  1190. );
  1191. });
  1192. }
  1193. /**
  1194. * Create YouTube videos
  1195. *
  1196. * @param {object} payload - an object containing the payload
  1197. * @param {string} payload.youtubeVideos - the youtubeVideo object or array of
  1198. * @returns {Promise} - returns a promise (resolve, reject)
  1199. */
  1200. CREATE_VIDEOS(payload) {
  1201. return new Promise((resolve, reject) => {
  1202. async.waterfall(
  1203. [
  1204. next => {
  1205. let { youtubeVideos } = payload;
  1206. if (typeof youtubeVideos !== "object") next("Invalid youtubeVideos type");
  1207. else {
  1208. if (!Array.isArray(youtubeVideos)) youtubeVideos = [youtubeVideos];
  1209. YouTubeModule.youtubeVideoModel.insertMany(youtubeVideos, next);
  1210. }
  1211. },
  1212. (youtubeVideos, next) => {
  1213. const youtubeIds = youtubeVideos.map(video => video.youtubeId);
  1214. async.eachLimit(
  1215. youtubeIds,
  1216. 2,
  1217. (youtubeId, next) => {
  1218. MediaModule.runJob("RECALCULATE_RATINGS", { youtubeId }, this)
  1219. .then(() => next())
  1220. .catch(next);
  1221. },
  1222. err => {
  1223. if (err) next(err);
  1224. else next(null, youtubeVideos);
  1225. }
  1226. );
  1227. }
  1228. ],
  1229. (err, youtubeVideos) => {
  1230. if (err) reject(new Error(err));
  1231. else resolve({ youtubeVideos });
  1232. }
  1233. );
  1234. });
  1235. }
  1236. /**
  1237. * Get YouTube video
  1238. *
  1239. * @param {object} payload - an object containing the payload
  1240. * @param {string} payload.identifier - the youtube video ObjectId or YouTube ID
  1241. * @param {string} payload.createMissing - attempt to fetch and create video if not in db
  1242. * @returns {Promise} - returns a promise (resolve, reject)
  1243. */
  1244. GET_VIDEO(payload) {
  1245. return new Promise((resolve, reject) => {
  1246. async.waterfall(
  1247. [
  1248. next => {
  1249. const query = mongoose.Types.ObjectId.isValid(payload.identifier)
  1250. ? { _id: payload.identifier }
  1251. : { youtubeId: payload.identifier };
  1252. return YouTubeModule.youtubeVideoModel.findOne(query, next);
  1253. },
  1254. (video, next) => {
  1255. if (video) return next(null, video, false);
  1256. if (mongoose.Types.ObjectId.isValid(payload.identifier) || !payload.createMissing)
  1257. return next("YouTube video not found.");
  1258. const params = {
  1259. part: "snippet,contentDetails,statistics,status",
  1260. id: payload.identifier
  1261. };
  1262. return YouTubeModule.runJob("API_GET_VIDEOS", { params }, this)
  1263. .then(({ response }) => {
  1264. const { data } = response;
  1265. if (data.items[0] === undefined)
  1266. return next("The specified video does not exist or cannot be publicly accessed.");
  1267. // TODO Clean up duration converter
  1268. let dur = data.items[0].contentDetails.duration;
  1269. dur = dur.replace("PT", "");
  1270. let duration = 0;
  1271. dur = dur.replace(/([\d]*)H/, (v, v2) => {
  1272. v2 = Number(v2);
  1273. duration = v2 * 60 * 60;
  1274. return "";
  1275. });
  1276. dur = dur.replace(/([\d]*)M/, (v, v2) => {
  1277. v2 = Number(v2);
  1278. duration += v2 * 60;
  1279. return "";
  1280. });
  1281. dur.replace(/([\d]*)S/, (v, v2) => {
  1282. v2 = Number(v2);
  1283. duration += v2;
  1284. return "";
  1285. });
  1286. const youtubeVideo = {
  1287. youtubeId: data.items[0].id,
  1288. title: data.items[0].snippet.title,
  1289. author: data.items[0].snippet.channelTitle,
  1290. thumbnail: data.items[0].snippet.thumbnails.default.url,
  1291. duration,
  1292. uploadedAt: new Date(data.items[0].snippet.publishedAt)
  1293. };
  1294. return next(null, false, youtubeVideo);
  1295. })
  1296. .catch(next);
  1297. },
  1298. (video, youtubeVideo, next) => {
  1299. if (video) return next(null, video, true);
  1300. return YouTubeModule.runJob("CREATE_VIDEOS", { youtubeVideos: youtubeVideo }, this)
  1301. .then(res => {
  1302. if (res.youtubeVideos.length === 1) next(null, res.youtubeVideos[0], false);
  1303. else next("YouTube video not found.");
  1304. })
  1305. .catch(next);
  1306. }
  1307. ],
  1308. (err, video, existing) => {
  1309. if (err) reject(new Error(err));
  1310. else resolve({ video, existing });
  1311. }
  1312. );
  1313. });
  1314. }
  1315. /**
  1316. * Remove YouTube videos
  1317. *
  1318. * @param {object} payload - an object containing the payload
  1319. * @param {string} payload.videoIds - Array of youtubeVideo ObjectIds
  1320. * @returns {Promise} - returns a promise (resolve, reject)
  1321. */
  1322. REMOVE_VIDEOS(payload) {
  1323. return new Promise((resolve, reject) => {
  1324. let { videoIds } = payload;
  1325. if (!Array.isArray(videoIds)) videoIds = [videoIds];
  1326. async.waterfall(
  1327. [
  1328. next => {
  1329. if (!videoIds.every(videoId => mongoose.Types.ObjectId.isValid(videoId)))
  1330. next("One or more videoIds are not a valid ObjectId.");
  1331. else {
  1332. this.publishProgress({ status: "update", message: `Removing video (stage 1)` });
  1333. YouTubeModule.youtubeVideoModel.find({ _id: { $in: videoIds } }, (err, videos) => {
  1334. if (err) next(err);
  1335. else
  1336. next(
  1337. null,
  1338. videos.map(video => video.youtubeId)
  1339. );
  1340. });
  1341. }
  1342. },
  1343. (youtubeIds, next) => {
  1344. this.publishProgress({ status: "update", message: `Removing video (stage 2)` });
  1345. SongsModule.SongModel.find({ youtubeId: { $in: youtubeIds } }, (err, songs) => {
  1346. if (err) next(err);
  1347. else {
  1348. const filteredIds = youtubeIds.filter(
  1349. youtubeId => !songs.find(song => song.youtubeId === youtubeId)
  1350. );
  1351. if (filteredIds.length < youtubeIds.length)
  1352. next("One or more videos are attached to songs.");
  1353. else next(null, filteredIds);
  1354. }
  1355. });
  1356. },
  1357. (youtubeIds, next) => {
  1358. this.publishProgress({ status: "update", message: `Removing video (stage 3)` });
  1359. MediaModule.runJob("REMOVE_RATINGS", { youtubeIds }, this)
  1360. .then(() => next(null, youtubeIds))
  1361. .catch(next);
  1362. },
  1363. (youtubeIds, next) => {
  1364. this.publishProgress({ status: "update", message: `Removing video (stage 4)` });
  1365. async.eachLimit(
  1366. youtubeIds,
  1367. 2,
  1368. (youtubeId, next) => {
  1369. async.waterfall(
  1370. [
  1371. next => {
  1372. PlaylistsModule.playlistModel.find(
  1373. { "songs.youtubeId": youtubeId },
  1374. (err, playlists) => {
  1375. if (err) next(err);
  1376. else {
  1377. async.eachLimit(
  1378. playlists,
  1379. 1,
  1380. (playlist, next) => {
  1381. PlaylistsModule.runJob(
  1382. "REMOVE_FROM_PLAYLIST",
  1383. { playlistId: playlist._id, youtubeId },
  1384. this
  1385. )
  1386. .then(() => next())
  1387. .catch(next);
  1388. },
  1389. next
  1390. );
  1391. }
  1392. }
  1393. );
  1394. },
  1395. next => {
  1396. StationsModule.stationModel.find(
  1397. { "queue.youtubeId": youtubeId },
  1398. (err, stations) => {
  1399. if (err) next(err);
  1400. else {
  1401. async.eachLimit(
  1402. stations,
  1403. 1,
  1404. (station, next) => {
  1405. StationsModule.runJob(
  1406. "REMOVE_FROM_QUEUE",
  1407. { stationId: station._id, youtubeId },
  1408. this
  1409. )
  1410. .then(() => next())
  1411. .catch(err => {
  1412. if (
  1413. err === "Station not found" ||
  1414. err ===
  1415. "Song is not currently in the queue."
  1416. )
  1417. next();
  1418. else next(err);
  1419. });
  1420. },
  1421. next
  1422. );
  1423. }
  1424. }
  1425. );
  1426. },
  1427. next => {
  1428. StationsModule.stationModel.find(
  1429. { "currentSong.youtubeId": youtubeId },
  1430. (err, stations) => {
  1431. if (err) next(err);
  1432. else {
  1433. async.eachLimit(
  1434. stations,
  1435. 1,
  1436. (station, next) => {
  1437. StationsModule.runJob(
  1438. "SKIP_STATION",
  1439. { stationId: station._id, natural: false },
  1440. this
  1441. )
  1442. .then(() => {
  1443. next();
  1444. })
  1445. .catch(err => {
  1446. if (err.message === "Station not found.")
  1447. next();
  1448. else next(err);
  1449. });
  1450. },
  1451. next
  1452. );
  1453. }
  1454. }
  1455. );
  1456. }
  1457. ],
  1458. next
  1459. );
  1460. },
  1461. next
  1462. );
  1463. },
  1464. next => {
  1465. this.publishProgress({ status: "update", message: `Removing video (stage 5)` });
  1466. YouTubeModule.youtubeVideoModel.deleteMany({ _id: { $in: videoIds } }, next);
  1467. },
  1468. (res, next) => {
  1469. this.publishProgress({ status: "update", message: `Removing video (stage 6)` });
  1470. CacheModule.runJob("PUB", {
  1471. channel: "youtube.removeVideos",
  1472. value: videoIds
  1473. })
  1474. .then(next)
  1475. .catch(err => next(err));
  1476. }
  1477. ],
  1478. err => {
  1479. if (err) reject(new Error(err));
  1480. else resolve();
  1481. }
  1482. );
  1483. });
  1484. }
  1485. /**
  1486. * Request a set of YouTube videos
  1487. *
  1488. * @param {object} payload - an object containing the payload
  1489. * @param {string} payload.url - the url of the the YouTube playlist or channel
  1490. * @param {boolean} payload.musicOnly - whether to only get music from the playlist/channel
  1491. * @param {boolean} payload.returnVideos - whether to return videos
  1492. * @returns {Promise} - returns a promise (resolve, reject)
  1493. */
  1494. REQUEST_SET(payload) {
  1495. return new Promise((resolve, reject) => {
  1496. async.waterfall(
  1497. [
  1498. next => {
  1499. const playlistRegex = /[\\?&]list=([^&#]*)/;
  1500. const channelRegex =
  1501. /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/;
  1502. if (playlistRegex.exec(payload.url) || channelRegex.exec(payload.url))
  1503. YouTubeModule.runJob(
  1504. playlistRegex.exec(payload.url) ? "GET_PLAYLIST" : "GET_CHANNEL",
  1505. {
  1506. url: payload.url,
  1507. musicOnly: payload.musicOnly
  1508. },
  1509. this
  1510. )
  1511. .then(res => {
  1512. next(null, res.songs);
  1513. })
  1514. .catch(next);
  1515. else next("Invalid YouTube URL.");
  1516. },
  1517. (youtubeIds, next) => {
  1518. let successful = 0;
  1519. let failed = 0;
  1520. let alreadyInDatabase = 0;
  1521. let videos = {};
  1522. const successfulVideoIds = [];
  1523. const failedVideoIds = [];
  1524. if (youtubeIds.length === 0) next();
  1525. async.eachOfLimit(
  1526. youtubeIds,
  1527. 1,
  1528. (youtubeId, index, next2) => {
  1529. YouTubeModule.runJob("GET_VIDEO", { identifier: youtubeId, createMissing: true }, this)
  1530. .then(res => {
  1531. successful += 1;
  1532. successfulVideoIds.push(youtubeId);
  1533. if (res.existing) alreadyInDatabase += 1;
  1534. if (res.video) videos[index] = res.video;
  1535. })
  1536. .catch(() => {
  1537. failed += 1;
  1538. failedVideoIds.push(youtubeId);
  1539. })
  1540. .finally(() => {
  1541. next2();
  1542. });
  1543. },
  1544. () => {
  1545. if (payload.returnVideos)
  1546. videos = Object.keys(videos)
  1547. .sort()
  1548. .map(key => videos[key]);
  1549. next(null, {
  1550. successful,
  1551. failed,
  1552. alreadyInDatabase,
  1553. videos,
  1554. successfulVideoIds,
  1555. failedVideoIds
  1556. });
  1557. }
  1558. );
  1559. }
  1560. ],
  1561. (err, response) => {
  1562. if (err) reject(new Error(err));
  1563. else resolve(response);
  1564. }
  1565. );
  1566. });
  1567. }
  1568. }
  1569. export default new _YouTubeModule();