youtube.js 52 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872
  1. import mongoose from "mongoose";
  2. import async from "async";
  3. import config from "config";
  4. import * as rax from "retry-axios";
  5. import axios from "axios";
  6. import CoreClass from "../core";
  7. class RateLimitter {
  8. /**
  9. * Constructor
  10. *
  11. * @param {number} timeBetween - The time between each allowed YouTube request
  12. */
  13. constructor(timeBetween) {
  14. this.dateStarted = Date.now();
  15. this.timeBetween = timeBetween;
  16. }
  17. /**
  18. * Returns a promise that resolves whenever the ratelimit of a YouTube request is done
  19. *
  20. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  21. */
  22. continue() {
  23. return new Promise(resolve => {
  24. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  25. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  26. });
  27. }
  28. /**
  29. * Restart the rate limit timer
  30. */
  31. restart() {
  32. this.dateStarted = Date.now();
  33. }
  34. }
  35. let YouTubeModule;
  36. let CacheModule;
  37. let DBModule;
  38. let MediaModule;
  39. let SongsModule;
  40. let StationsModule;
  41. let PlaylistsModule;
  42. let WSModule;
  43. const isQuotaExceeded = apiCalls => {
  44. const reversedApiCalls = apiCalls.slice().reverse();
  45. const quotas = config.get("apis.youtube.quotas").slice();
  46. const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
  47. let quotaExceeded = false;
  48. sortedQuotas.forEach(quota => {
  49. let quotaUsed = 0;
  50. let dateCutoff = null;
  51. if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date() - 1000 * 60;
  52. else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date() - 1000 * 100;
  53. else if (quota.type === "QUERIES_PER_DAY") {
  54. // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
  55. dateCutoff = new Date();
  56. dateCutoff.setUTCMilliseconds(0);
  57. dateCutoff.setUTCSeconds(0);
  58. dateCutoff.setUTCMinutes(0);
  59. dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
  60. dateCutoff.setUTCHours(0);
  61. }
  62. reversedApiCalls.forEach(apiCall => {
  63. if (apiCall.date >= dateCutoff) quotaUsed += apiCall.quotaCost;
  64. });
  65. if (quotaUsed >= quota.limit) {
  66. quotaExceeded = true;
  67. }
  68. });
  69. return quotaExceeded;
  70. };
  71. class _YouTubeModule extends CoreClass {
  72. // eslint-disable-next-line require-jsdoc
  73. constructor() {
  74. super("youtube", {
  75. concurrency: 10,
  76. priorities: {
  77. GET_PLAYLIST: 11
  78. }
  79. });
  80. YouTubeModule = this;
  81. }
  82. /**
  83. * Initialises the activities module
  84. *
  85. * @returns {Promise} - returns promise (reject, resolve)
  86. */
  87. async initialize() {
  88. CacheModule = this.moduleManager.modules.cache;
  89. DBModule = this.moduleManager.modules.db;
  90. MediaModule = this.moduleManager.modules.media;
  91. SongsModule = this.moduleManager.modules.songs;
  92. StationsModule = this.moduleManager.modules.stations;
  93. PlaylistsModule = this.moduleManager.modules.playlists;
  94. WSModule = this.moduleManager.modules.ws;
  95. this.youtubeApiRequestModel = this.YoutubeApiRequestModel = await DBModule.runJob("GET_MODEL", {
  96. modelName: "youtubeApiRequest"
  97. });
  98. this.youtubeVideoModel = this.YoutubeVideoModel = await DBModule.runJob("GET_MODEL", {
  99. modelName: "youtubeVideo"
  100. });
  101. this.youtubeChannelModel = this.YoutubeChannelModel = await DBModule.runJob("GET_MODEL", {
  102. modelName: "youtubeChannel"
  103. });
  104. return new Promise(resolve => {
  105. CacheModule.runJob("SUB", {
  106. channel: "youtube.removeYoutubeApiRequest",
  107. cb: requestId => {
  108. WSModule.runJob("EMIT_TO_ROOM", {
  109. room: `view-api-request.${requestId}`,
  110. args: ["event:youtubeApiRequest.removed"]
  111. });
  112. WSModule.runJob("EMIT_TO_ROOM", {
  113. room: "admin.youtube",
  114. args: ["event:admin.youtubeApiRequest.removed", { data: { requestId } }]
  115. });
  116. }
  117. });
  118. CacheModule.runJob("SUB", {
  119. channel: "youtube.removeVideos",
  120. cb: videoIds => {
  121. const videos = Array.isArray(videoIds) ? videoIds : [videoIds];
  122. videos.forEach(videoId => {
  123. WSModule.runJob("EMIT_TO_ROOM", {
  124. room: `view-media.youtube:${videoId}`,
  125. args: ["event:youtubeVideo.removed"]
  126. });
  127. WSModule.runJob("EMIT_TO_ROOM", {
  128. room: "admin.youtubeVideos",
  129. args: ["event:admin.youtubeVideo.removed", { data: { videoId } }]
  130. });
  131. WSModule.runJob("EMIT_TO_ROOMS", {
  132. rooms: ["import-album", "edit-songs"],
  133. args: ["event:admin.youtubeVideo.removed", { videoId }]
  134. });
  135. });
  136. }
  137. });
  138. this.rateLimiter = new RateLimitter(config.get("apis.youtube.rateLimit"));
  139. this.requestTimeout = config.get("apis.youtube.requestTimeout");
  140. this.axios = axios.create();
  141. this.axios.defaults.raxConfig = {
  142. instance: this.axios,
  143. retry: config.get("apis.youtube.retryAmount"),
  144. noResponseRetries: config.get("apis.youtube.retryAmount")
  145. };
  146. rax.attach(this.axios);
  147. this.youtubeApiRequestModel
  148. .find(
  149. { date: { $gte: new Date() - 2 * 24 * 60 * 60 * 1000 } },
  150. { date: true, quotaCost: true, _id: false }
  151. )
  152. .sort({ date: 1 })
  153. .exec((err, youtubeApiRequests) => {
  154. if (err) console.log("Couldn't load YouTube API requests.");
  155. else {
  156. this.apiCalls = youtubeApiRequests;
  157. resolve();
  158. }
  159. });
  160. });
  161. }
  162. /**
  163. * Fetches a list of songs from Youtube's API
  164. *
  165. * @param {object} payload - object that contains the payload
  166. * @param {string} payload.query - the query we'll pass to youtubes api
  167. * @param {string} payload.pageToken - (optional) if this exists, will search search youtube for a specific page reference
  168. * @returns {Promise} - returns promise (reject, resolve)
  169. */
  170. SEARCH(payload) {
  171. const params = {
  172. part: "snippet",
  173. q: payload.query,
  174. type: "video",
  175. maxResults: 10
  176. };
  177. if (payload.pageToken) params.pageToken = payload.pageToken;
  178. return new Promise((resolve, reject) => {
  179. YouTubeModule.runJob(
  180. "API_SEARCH",
  181. {
  182. params
  183. },
  184. this
  185. )
  186. .then(({ response }) => {
  187. const { data } = response;
  188. return resolve(data);
  189. })
  190. .catch(err => {
  191. YouTubeModule.log("ERROR", "SEARCH", `${err.message}`);
  192. if (err.message === "Searching with YouTube is disabled.") return reject(err);
  193. return reject(new Error("An error has occured. Please try again later."));
  194. });
  195. });
  196. }
  197. /**
  198. * Returns details about the YouTube quota usage
  199. *
  200. * @param {object} payload - object that contains the payload
  201. * @param {string} payload.fromDate - date to select requests up to
  202. * @returns {Promise} - returns promise (reject, resolve)
  203. */
  204. GET_QUOTA_STATUS(payload) {
  205. return new Promise((resolve, reject) => {
  206. const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
  207. YouTubeModule.youtubeApiRequestModel
  208. .find(
  209. { date: { $gte: fromDate - 2 * 24 * 60 * 60 * 1000, $lte: fromDate } },
  210. { date: true, quotaCost: true, _id: false }
  211. )
  212. .sort({ date: 1 })
  213. .exec((err, youtubeApiRequests) => {
  214. if (err) reject(new Error("Couldn't load YouTube API requests."));
  215. else {
  216. const reversedApiCalls = youtubeApiRequests.slice().reverse();
  217. const quotas = config.get("apis.youtube.quotas").slice();
  218. const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
  219. const status = {};
  220. sortedQuotas.forEach(quota => {
  221. status[quota.type] = {
  222. title: quota.title,
  223. quotaUsed: 0,
  224. limit: quota.limit,
  225. quotaExceeded: false
  226. };
  227. let dateCutoff = null;
  228. if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date(fromDate) - 1000 * 60;
  229. else if (quota.type === "QUERIES_PER_100_SECONDS")
  230. dateCutoff = new Date(fromDate) - 1000 * 100;
  231. else if (quota.type === "QUERIES_PER_DAY") {
  232. // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
  233. dateCutoff = new Date(fromDate);
  234. dateCutoff.setUTCMilliseconds(0);
  235. dateCutoff.setUTCSeconds(0);
  236. dateCutoff.setUTCMinutes(0);
  237. dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
  238. dateCutoff.setUTCHours(0);
  239. }
  240. reversedApiCalls.forEach(apiCall => {
  241. if (apiCall.date >= dateCutoff) status[quota.type].quotaUsed += apiCall.quotaCost;
  242. });
  243. if (status[quota.type].quotaUsed >= quota.limit && !status[quota.type].quotaExceeded)
  244. status[quota.type].quotaExceeded = true;
  245. });
  246. resolve({ status });
  247. }
  248. });
  249. });
  250. }
  251. /**
  252. * Returns YouTube quota chart data
  253. *
  254. * @param {object} payload - object that contains the payload
  255. * @param {string} payload.timePeriod - either hours or days
  256. * @param {string} payload.startDate - beginning date
  257. * @param {string} payload.endDate - end date
  258. * @param {string} payload.dataType - either usage or count
  259. * @returns {Promise} - returns promise (reject, resolve)
  260. */
  261. GET_QUOTA_CHART_DATA(payload) {
  262. return new Promise((resolve, reject) => {
  263. const { timePeriod, startDate, endDate, dataType } = payload;
  264. // const timePeriod = "hours";
  265. // const startDate = new Date("2022-05-20 00:00:00");
  266. // const endDate = new Date("2022-05-21 00:00:00");
  267. // const timePeriod = "days";
  268. // const startDate = new Date("2022-05-15 00:00:00");
  269. // const endDate = new Date("2022-05-21 00:00:00");
  270. // const endDate = new Date("2022-05-30 00:00:00");
  271. // const dataType = "usage";
  272. // const dataType = "count";
  273. async.waterfall(
  274. [
  275. next => {
  276. let timeRanges = [];
  277. if (timePeriod === "hours") {
  278. startDate.setMinutes(0, 0, 0);
  279. endDate.setMinutes(0, 0, 0);
  280. const lastDate = new Date(startDate);
  281. do {
  282. const newDate = new Date(lastDate.getTime() + 1000 * 60 * 60);
  283. timeRanges.push({
  284. startDate: new Date(lastDate),
  285. endDate: newDate
  286. });
  287. lastDate.setTime(newDate.getTime());
  288. } while (lastDate.getTime() < endDate.getTime());
  289. if (timeRanges.length === 0 || timeRanges.length > 24)
  290. return next("No valid time ranges specified.");
  291. timeRanges = timeRanges.map(timeRange => ({
  292. ...timeRange,
  293. label: `${timeRange.startDate.getHours().toString().padStart(2, "0")}:00`
  294. }));
  295. } else if (timePeriod === "days") {
  296. startDate.setHours(0, 0, 0, 0);
  297. endDate.setHours(0, 0, 0, 0);
  298. const lastDate = new Date(startDate);
  299. do {
  300. const newDate = new Date(lastDate.getTime() + 1000 * 60 * 60 * 24);
  301. timeRanges.push({
  302. startDate: new Date(lastDate),
  303. endDate: newDate
  304. });
  305. lastDate.setTime(newDate.getTime());
  306. } while (lastDate.getTime() < endDate.getTime());
  307. if (timeRanges.length === 0 || timeRanges.length > 31)
  308. return next("No valid time ranges specified.");
  309. const days = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
  310. if (timeRanges.length <= 7)
  311. timeRanges = timeRanges.map(timeRange => ({
  312. ...timeRange,
  313. label: days[timeRange.startDate.getDay()]
  314. }));
  315. else
  316. timeRanges = timeRanges.map(timeRange => ({
  317. ...timeRange,
  318. label: `${timeRange.startDate.getDate().toString().padStart(2, "0")}-${(
  319. timeRange.startDate.getMonth() + 1
  320. )
  321. .toString()
  322. .padStart(2, "0")}`
  323. }));
  324. }
  325. return next(null, timeRanges);
  326. },
  327. (timeRanges, next) => {
  328. YouTubeModule.youtubeApiRequestModel
  329. .find({
  330. date: { $gte: startDate, $lte: endDate }
  331. })
  332. .sort({ date: 1 })
  333. .exec((err, youtubeApiRequests) => {
  334. next(err, timeRanges, youtubeApiRequests);
  335. });
  336. },
  337. (timeRanges, youtubeApiRequests, next) => {
  338. const regex = /https:\/\/www\.googleapis\.com\/youtube\/v3\/(.+)/;
  339. const requestTypes = Object.fromEntries(
  340. youtubeApiRequests
  341. .map(youtubeApiRequest => regex.exec(youtubeApiRequest.url)[1])
  342. .filter((requestType, index, array) => array.indexOf(requestType) === index)
  343. .map(requestType => [requestType, 0])
  344. );
  345. timeRanges = timeRanges.map(timeRange => ({
  346. ...timeRange,
  347. data: { total: 0, ...requestTypes }
  348. }));
  349. youtubeApiRequests.forEach(youtubeApiRequest => {
  350. timeRanges.forEach(timeRange => {
  351. if (
  352. timeRange.startDate <= youtubeApiRequest.date &&
  353. timeRange.endDate >= youtubeApiRequest.date
  354. ) {
  355. const requestType = regex.exec(youtubeApiRequest.url)[1];
  356. if (!timeRange.data[requestType]) timeRange.data[requestType] = 0;
  357. if (dataType === "usage") {
  358. timeRange.data[requestType] += youtubeApiRequest.quotaCost;
  359. timeRange.data.total += youtubeApiRequest.quotaCost;
  360. } else if (dataType === "count") {
  361. timeRange.data[requestType] += 1;
  362. timeRange.data.total += 1;
  363. }
  364. }
  365. });
  366. });
  367. next(null, timeRanges);
  368. },
  369. (timeRanges, next) => {
  370. const chartData = {};
  371. chartData.labels = timeRanges.map(timeRange => timeRange.label);
  372. const datasetTypes = Object.keys(timeRanges[0].data);
  373. const colors = {
  374. total: "rgb(2, 166, 242)",
  375. videos: "rgb(166, 2, 242)",
  376. search: "rgb(242, 2, 166)",
  377. channels: "rgb(2, 242, 166)",
  378. playlistItems: "rgb(242, 166, 2)"
  379. };
  380. chartData.datasets = datasetTypes.map(datasetType => {
  381. let label;
  382. switch (datasetType) {
  383. case "total":
  384. label = "Total";
  385. break;
  386. case "videos":
  387. label = "Videos";
  388. break;
  389. case "search":
  390. label = "Search";
  391. break;
  392. case "playlistItems":
  393. label = "Playlist Items";
  394. break;
  395. default:
  396. label = datasetType;
  397. }
  398. return {
  399. label,
  400. borderColor: colors[datasetType],
  401. backgroundColor: colors[datasetType],
  402. data: timeRanges.map(timeRange => timeRange.data[datasetType])
  403. };
  404. });
  405. next(null, chartData);
  406. }
  407. ],
  408. (err, chartData) => {
  409. if (err) reject(err);
  410. else resolve(chartData);
  411. }
  412. );
  413. });
  414. }
  415. /**
  416. * Gets the id of the channel upload playlist
  417. *
  418. * @param {object} payload - object that contains the payload
  419. * @param {string} payload.id - the id of the YouTube channel. Optional: can be left out if specifying a username.
  420. * @param {string} payload.username - the username of the YouTube channel. Only gets used if no id is specified.
  421. * @returns {Promise} - returns promise (reject, resolve)
  422. */
  423. GET_CHANNEL_UPLOADS_PLAYLIST_ID(payload) {
  424. return new Promise((resolve, reject) => {
  425. const params = {
  426. part: "id,contentDetails"
  427. };
  428. if (payload.id) params.id = payload.id;
  429. else params.forUsername = payload.username;
  430. YouTubeModule.runJob(
  431. "API_GET_CHANNELS",
  432. {
  433. params
  434. },
  435. this
  436. )
  437. .then(({ response }) => {
  438. const { data } = response;
  439. if (data.pageInfo.totalResults === 0) return reject(new Error("Channel not found."));
  440. const playlistId = data.items[0].contentDetails.relatedPlaylists.uploads;
  441. return resolve({ playlistId });
  442. })
  443. .catch(err => {
  444. YouTubeModule.log("ERROR", "GET_CHANNEL_UPLOADS_PLAYLIST_ID", `${err.message}`);
  445. if (err.message === "Request failed with status code 404")
  446. return reject(new Error("Channel not found. Is the channel public/unlisted?"));
  447. if (err.message === "Searching with YouTube is disabled.") return reject(err);
  448. return reject(new Error("An error has occured. Please try again later."));
  449. });
  450. });
  451. }
  452. /**
  453. * Gets the id of the channel from the custom URL
  454. *
  455. * @param {object} payload - object that contains the payload
  456. * @param {string} payload.customUrl - the customUrl of the YouTube channel
  457. * @returns {Promise} - returns promise (reject, resolve)
  458. */
  459. GET_CHANNEL_ID_FROM_CUSTOM_URL(payload) {
  460. return new Promise((resolve, reject) => {
  461. async.waterfall(
  462. [
  463. next => {
  464. const params = {
  465. part: "snippet",
  466. type: "channel",
  467. maxResults: 50
  468. };
  469. params.q = payload.customUrl;
  470. YouTubeModule.runJob(
  471. "API_SEARCH",
  472. {
  473. params
  474. },
  475. this
  476. )
  477. .then(({ response }) => {
  478. const { data } = response;
  479. if (data.pageInfo.totalResults === 0) return next("Channel not found.");
  480. const channelIds = data.items.map(item => item.id.channelId);
  481. return next(null, channelIds);
  482. })
  483. .catch(err => {
  484. next(err);
  485. });
  486. },
  487. (channelIds, next) => {
  488. const params = {
  489. part: "snippet",
  490. id: channelIds.join(","),
  491. maxResults: 50
  492. };
  493. YouTubeModule.runJob(
  494. "API_GET_CHANNELS",
  495. {
  496. params
  497. },
  498. this
  499. )
  500. .then(({ response }) => {
  501. const { data } = response;
  502. if (data.pageInfo.totalResults === 0) return next("Channel not found.");
  503. let channelId = null;
  504. data.items.forEach(item => {
  505. if (
  506. item.snippet.customUrl &&
  507. item.snippet.customUrl.toLowerCase() === payload.customUrl.toLowerCase()
  508. ) {
  509. channelId = item.id;
  510. }
  511. });
  512. if (!channelId) return next("Channel not found.");
  513. return next(null, channelId);
  514. })
  515. .catch(err => {
  516. next(err);
  517. });
  518. }
  519. ],
  520. (err, channelId) => {
  521. if (err) {
  522. YouTubeModule.log("ERROR", "GET_CHANNEL_ID_FROM_CUSTOM_URL", `${err.message || err}`);
  523. if (err.message === "Request failed with status code 404")
  524. return reject(new Error("Channel not found. Is the channel public/unlisted?"));
  525. if (err.message === "Searching with YouTube is disabled.") return reject(err);
  526. return reject(new Error("An error has occured. Please try again later."));
  527. }
  528. return resolve({ channelId });
  529. }
  530. );
  531. });
  532. }
  533. /**
  534. * Returns an array of songs taken from a YouTube playlist
  535. *
  536. * @param {object} payload - object that contains the payload
  537. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the playlist
  538. * @param {string} payload.url - the url of the YouTube playlist
  539. * @returns {Promise} - returns promise (reject, resolve)
  540. */
  541. GET_PLAYLIST(payload) {
  542. return new Promise((resolve, reject) => {
  543. const regex = /[\\?&]list=([^&#]*)/;
  544. const splitQuery = regex.exec(payload.url);
  545. if (!splitQuery) {
  546. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Invalid YouTube playlist URL query.");
  547. reject(new Error("Invalid playlist URL."));
  548. return;
  549. }
  550. const playlistId = splitQuery[1];
  551. const maxPages = Number.parseInt(config.get("apis.youtube.maxPlaylistPages"));
  552. let currentPage = 0;
  553. async.waterfall(
  554. [
  555. next => {
  556. let songs = [];
  557. let nextPageToken = "";
  558. async.whilst(
  559. next => {
  560. YouTubeModule.log(
  561. "INFO",
  562. `Getting playlist progress for job (${this.toString()}): ${
  563. songs.length
  564. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  565. );
  566. next(null, nextPageToken !== undefined && currentPage < maxPages);
  567. },
  568. next => {
  569. currentPage += 1;
  570. // Add 250ms delay between each job request
  571. setTimeout(() => {
  572. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  573. .then(response => {
  574. songs = songs.concat(response.songs);
  575. nextPageToken = response.nextPageToken;
  576. next();
  577. })
  578. .catch(err => next(err));
  579. }, 250);
  580. },
  581. err => next(err, songs)
  582. );
  583. },
  584. (songs, next) =>
  585. next(
  586. null,
  587. songs.map(song => song.contentDetails.videoId)
  588. ),
  589. (songs, next) => {
  590. if (!payload.musicOnly) return next(true, { songs });
  591. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  592. .then(filteredSongs => next(null, { filteredSongs, songs }))
  593. .catch(next);
  594. }
  595. ],
  596. (err, response) => {
  597. if (err && err !== true) {
  598. YouTubeModule.log("ERROR", "GET_PLAYLIST", "Some error has occurred.", err.message);
  599. reject(new Error(err.message));
  600. } else {
  601. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  602. }
  603. }
  604. );
  605. });
  606. }
  607. /**
  608. * Returns a a page from a YouTube playlist. Is used internally by GET_PLAYLIST and GET_CHANNEL_VIDEOS.
  609. *
  610. * @param {object} payload - object that contains the payload
  611. * @param {boolean} payload.playlistId - the playlist id to get videos from
  612. * @param {boolean} payload.nextPageToken - the nextPageToken to use
  613. * @param {string} payload.url - the url of the YouTube playlist
  614. * @returns {Promise} - returns promise (reject, resolve)
  615. */
  616. GET_PLAYLIST_PAGE(payload) {
  617. return new Promise((resolve, reject) => {
  618. const params = {
  619. part: "contentDetails",
  620. playlistId: payload.playlistId,
  621. maxResults: 50
  622. };
  623. if (payload.nextPageToken) params.pageToken = payload.nextPageToken;
  624. YouTubeModule.runJob(
  625. "API_GET_PLAYLIST_ITEMS",
  626. {
  627. params
  628. },
  629. this
  630. )
  631. .then(({ response }) => {
  632. const { data } = response;
  633. const songs = data.items;
  634. if (data.nextPageToken) return resolve({ nextPageToken: data.nextPageToken, songs });
  635. return resolve({ songs });
  636. })
  637. .catch(err => {
  638. YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${err.message}`);
  639. if (err.message === "Request failed with status code 404")
  640. return reject(new Error("Playlist not found. Is the playlist public/unlisted?"));
  641. return reject(new Error("An error has occured. Please try again later."));
  642. });
  643. });
  644. }
  645. /**
  646. * Filters a list of YouTube videos so that they only contains videos with music. Is used internally by GET_PLAYLIST
  647. *
  648. * @param {object} payload - object that contains the payload
  649. * @param {Array} payload.videoIds - an array of YouTube videoIds to filter through
  650. * @param {Array} payload.page - the current page/set of video's to get, starting at 0. If left null, 0 is assumed. Will recurse.
  651. * @returns {Promise} - returns promise (reject, resolve)
  652. */
  653. FILTER_MUSIC_VIDEOS(payload) {
  654. return new Promise((resolve, reject) => {
  655. const page = payload.page ? payload.page : 0;
  656. const videosPerPage = 50;
  657. const localVideoIds = payload.videoIds.splice(page * 50, videosPerPage);
  658. if (localVideoIds.length === 0) {
  659. resolve({ videoIds: [] });
  660. return;
  661. }
  662. const params = {
  663. part: "topicDetails",
  664. id: localVideoIds.join(","),
  665. maxResults: videosPerPage
  666. };
  667. YouTubeModule.runJob("API_GET_VIDEOS", { params }, this)
  668. .then(({ response }) => {
  669. const { data } = response;
  670. const videoIds = [];
  671. data.items.forEach(item => {
  672. const videoId = item.id;
  673. if (!item.topicDetails) return;
  674. if (item.topicDetails.topicCategories.indexOf("https://en.wikipedia.org/wiki/Music") !== -1)
  675. videoIds.push(videoId);
  676. });
  677. return YouTubeModule.runJob(
  678. "FILTER_MUSIC_VIDEOS",
  679. { videoIds: payload.videoIds, page: page + 1 },
  680. this
  681. )
  682. .then(result => resolve({ videoIds: videoIds.concat(result.videoIds) }))
  683. .catch(err => reject(err));
  684. })
  685. .catch(err => {
  686. YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${err.message}`);
  687. return reject(new Error("Failed to find playlist from YouTube"));
  688. });
  689. });
  690. }
  691. /**
  692. * Returns an array of songs taken from a YouTube channel
  693. *
  694. * @param {object} payload - object that contains the payload
  695. * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the channel
  696. * @param {boolean} payload.disableSearch - whether to allow searching for custom url/username
  697. * @param {string} payload.url - the url of the YouTube channel
  698. * @returns {Promise} - returns promise (reject, resolve)
  699. */
  700. GET_CHANNEL_VIDEOS(payload) {
  701. return new Promise((resolve, reject) => {
  702. const regex =
  703. /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/;
  704. const splitQuery = regex.exec(payload.url);
  705. if (!splitQuery) {
  706. YouTubeModule.log("ERROR", "GET_CHANNEL", "Invalid YouTube channel URL query.");
  707. reject(new Error("Invalid playlist URL."));
  708. return;
  709. }
  710. const channelId = splitQuery[1];
  711. const channelUsername = splitQuery[2];
  712. const channelCustomUrl = splitQuery[3];
  713. const channelUsernameOrCustomUrl = splitQuery[4];
  714. const disableSearch = payload.disableSearch || false;
  715. async.waterfall(
  716. [
  717. next => {
  718. const payload = {};
  719. if (channelId) payload.id = channelId;
  720. else if (channelUsername) payload.username = channelUsername;
  721. else return next(null, true, null);
  722. return YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", payload, this)
  723. .then(({ playlistId }) => {
  724. next(null, false, playlistId);
  725. })
  726. .catch(err => {
  727. if (err.message === "Channel not found. Is the channel public/unlisted?")
  728. next(null, true, null);
  729. else next(err);
  730. });
  731. },
  732. (getUsernameFromCustomUrl, playlistId, next) => {
  733. if (!getUsernameFromCustomUrl) return next(null, playlistId);
  734. if (disableSearch)
  735. return next(
  736. "Importing with this type of URL is disabled. Please provide a channel URL with the channel ID."
  737. );
  738. const payload = {};
  739. if (channelCustomUrl) payload.customUrl = channelCustomUrl;
  740. else if (channelUsernameOrCustomUrl) payload.customUrl = channelUsernameOrCustomUrl;
  741. else return next("No proper URL provided.");
  742. return YouTubeModule.runJob("GET_CHANNEL_ID_FROM_CUSTOM_URL", payload, this)
  743. .then(({ channelId }) => {
  744. YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", { id: channelId }, this)
  745. .then(({ playlistId }) => {
  746. next(null, playlistId);
  747. })
  748. .catch(err => next(err));
  749. })
  750. .catch(err => next(err));
  751. },
  752. (playlistId, next) => {
  753. let songs = [];
  754. let nextPageToken = "";
  755. async.whilst(
  756. next => {
  757. YouTubeModule.log(
  758. "INFO",
  759. `Getting channel progress for job (${this.toString()}): ${
  760. songs.length
  761. } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
  762. );
  763. next(null, nextPageToken !== undefined);
  764. },
  765. next => {
  766. // Add 250ms delay between each job request
  767. setTimeout(() => {
  768. YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this)
  769. .then(response => {
  770. songs = songs.concat(response.songs);
  771. nextPageToken = response.nextPageToken;
  772. next();
  773. })
  774. .catch(err => next(err));
  775. }, 250);
  776. },
  777. err => next(err, songs)
  778. );
  779. },
  780. (songs, next) =>
  781. next(
  782. null,
  783. songs.map(song => song.contentDetails.videoId)
  784. ),
  785. (songs, next) => {
  786. if (!payload.musicOnly) return next(true, { songs });
  787. return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this)
  788. .then(filteredSongs => next(null, { filteredSongs, songs }))
  789. .catch(next);
  790. }
  791. ],
  792. (err, response) => {
  793. if (err && err !== true) {
  794. YouTubeModule.log("ERROR", "GET_CHANNEL", "Some error has occurred.", err.message || err);
  795. reject(new Error(err.message || err));
  796. } else {
  797. resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs });
  798. }
  799. }
  800. );
  801. });
  802. }
  803. /**
  804. * Perform YouTube API get videos request
  805. *
  806. * @param {object} payload - object that contains the payload
  807. * @param {object} payload.params - request parameters
  808. * @returns {Promise} - returns promise (reject, resolve)
  809. */
  810. API_GET_VIDEOS(payload) {
  811. return new Promise((resolve, reject) => {
  812. const { params } = payload;
  813. YouTubeModule.runJob(
  814. "API_CALL",
  815. {
  816. url: "https://www.googleapis.com/youtube/v3/videos",
  817. params: {
  818. key: config.get("apis.youtube.key"),
  819. ...params
  820. },
  821. quotaCost: 1
  822. },
  823. this
  824. )
  825. .then(response => {
  826. resolve(response);
  827. })
  828. .catch(err => {
  829. reject(err);
  830. });
  831. });
  832. }
  833. /**
  834. * Perform YouTube API get playlist items request
  835. *
  836. * @param {object} payload - object that contains the payload
  837. * @param {object} payload.params - request parameters
  838. * @returns {Promise} - returns promise (reject, resolve)
  839. */
  840. API_GET_PLAYLIST_ITEMS(payload) {
  841. return new Promise((resolve, reject) => {
  842. const { params } = payload;
  843. YouTubeModule.runJob(
  844. "API_CALL",
  845. {
  846. url: "https://www.googleapis.com/youtube/v3/playlistItems",
  847. params: {
  848. key: config.get("apis.youtube.key"),
  849. ...params
  850. },
  851. quotaCost: 1
  852. },
  853. this
  854. )
  855. .then(response => {
  856. resolve(response);
  857. })
  858. .catch(err => {
  859. reject(err);
  860. });
  861. });
  862. }
  863. /**
  864. * Perform YouTube API get channels request
  865. *
  866. * @param {object} payload - object that contains the payload
  867. * @param {object} payload.params - request parameters
  868. * @returns {Promise} - returns promise (reject, resolve)
  869. */
  870. API_GET_CHANNELS(payload) {
  871. return new Promise((resolve, reject) => {
  872. const { params } = payload;
  873. YouTubeModule.runJob(
  874. "API_CALL",
  875. {
  876. url: "https://www.googleapis.com/youtube/v3/channels",
  877. params: {
  878. key: config.get("apis.youtube.key"),
  879. ...params
  880. },
  881. quotaCost: 1
  882. },
  883. this
  884. )
  885. .then(response => {
  886. resolve(response);
  887. })
  888. .catch(err => {
  889. reject(err);
  890. });
  891. });
  892. }
  893. /**
  894. * Perform YouTube API search request
  895. *
  896. * @param {object} payload - object that contains the payload
  897. * @param {object} payload.params - request parameters
  898. * @returns {Promise} - returns promise (reject, resolve)
  899. */
  900. API_SEARCH(payload) {
  901. return new Promise((resolve, reject) => {
  902. const { params } = payload;
  903. if (config.get("experimental.disable_youtube_search")) {
  904. reject(new Error("Searching with YouTube is disabled."));
  905. return;
  906. }
  907. YouTubeModule.runJob(
  908. "API_CALL",
  909. {
  910. url: "https://www.googleapis.com/youtube/v3/search",
  911. params: {
  912. key: config.get("apis.youtube.key"),
  913. ...params
  914. },
  915. quotaCost: 100
  916. },
  917. this
  918. )
  919. .then(response => {
  920. resolve(response);
  921. })
  922. .catch(err => {
  923. reject(err);
  924. });
  925. });
  926. }
  927. /**
  928. * Perform YouTube API call
  929. *
  930. * @param {object} payload - object that contains the payload
  931. * @param {object} payload.url - request url
  932. * @param {object} payload.params - request parameters
  933. * @param {object} payload.quotaCost - request quotaCost
  934. * @returns {Promise} - returns promise (reject, resolve)
  935. */
  936. API_CALL(payload) {
  937. return new Promise((resolve, reject) => {
  938. const { url, params, quotaCost } = payload;
  939. const quotaExceeded = isQuotaExceeded(YouTubeModule.apiCalls);
  940. if (quotaExceeded) reject(new Error("Quota has been exceeded. Please wait a while."));
  941. else {
  942. const youtubeApiRequest = new YouTubeModule.YoutubeApiRequestModel({
  943. url,
  944. date: Date.now(),
  945. quotaCost
  946. });
  947. youtubeApiRequest.save();
  948. const { ...keylessParams } = payload.params;
  949. CacheModule.runJob("HSET", {
  950. table: "youtubeApiRequestParams",
  951. key: youtubeApiRequest._id.toString(),
  952. value: JSON.stringify(keylessParams)
  953. }).then();
  954. YouTubeModule.apiCalls.push({ date: youtubeApiRequest.date, quotaCost });
  955. YouTubeModule.axios
  956. .get(url, {
  957. params,
  958. timeout: YouTubeModule.requestTimeout
  959. })
  960. .then(response => {
  961. if (response.data.error) {
  962. reject(new Error(response.data.error));
  963. } else {
  964. CacheModule.runJob("HSET", {
  965. table: "youtubeApiRequestResults",
  966. key: youtubeApiRequest._id.toString(),
  967. value: JSON.stringify(response.data)
  968. }).then();
  969. resolve({ response });
  970. }
  971. })
  972. .catch(err => {
  973. reject(err);
  974. });
  975. }
  976. });
  977. }
  978. /**
  979. * Fetch all api requests
  980. *
  981. * @param {object} payload - object that contains the payload
  982. * @param {object} payload.fromDate - data to fetch requests up to
  983. * @returns {Promise} - returns promise (reject, resolve)
  984. */
  985. GET_API_REQUESTS(payload) {
  986. return new Promise((resolve, reject) => {
  987. const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
  988. YouTubeModule.youtubeApiRequestModel
  989. .find({ date: { $lte: fromDate } })
  990. .sort({ date: -1 })
  991. .exec((err, youtubeApiRequests) => {
  992. if (err) reject(new Error("Couldn't load YouTube API requests."));
  993. else {
  994. resolve({ apiRequests: youtubeApiRequests });
  995. }
  996. });
  997. });
  998. }
  999. /**
  1000. * Fetch an api request
  1001. *
  1002. * @param {object} payload - object that contains the payload
  1003. * @param {object} payload.apiRequestId - the api request id
  1004. * @returns {Promise} - returns promise (reject, resolve)
  1005. */
  1006. GET_API_REQUEST(payload) {
  1007. return new Promise((resolve, reject) => {
  1008. const { apiRequestId } = payload;
  1009. async.waterfall(
  1010. [
  1011. next => {
  1012. YouTubeModule.youtubeApiRequestModel.findOne({ _id: apiRequestId }).exec(next);
  1013. },
  1014. (apiRequest, next) => {
  1015. CacheModule.runJob(
  1016. "HGET",
  1017. {
  1018. table: "youtubeApiRequestParams",
  1019. key: apiRequestId.toString()
  1020. },
  1021. this
  1022. )
  1023. .then(apiRequestParams => {
  1024. next(null, {
  1025. ...apiRequest._doc,
  1026. params: apiRequestParams
  1027. });
  1028. })
  1029. .catch(err => next(err));
  1030. },
  1031. (apiRequest, next) => {
  1032. CacheModule.runJob(
  1033. "HGET",
  1034. {
  1035. table: "youtubeApiRequestResults",
  1036. key: apiRequestId.toString()
  1037. },
  1038. this
  1039. )
  1040. .then(apiRequestResults => {
  1041. next(null, {
  1042. ...apiRequest,
  1043. results: apiRequestResults
  1044. });
  1045. })
  1046. .catch(err => next(err));
  1047. }
  1048. ],
  1049. (err, apiRequest) => {
  1050. if (err) reject(new Error(err));
  1051. else resolve({ apiRequest });
  1052. }
  1053. );
  1054. });
  1055. }
  1056. /**
  1057. * Removed all stored api requests from mongo and redis
  1058. *
  1059. * @returns {Promise} - returns promise (reject, resolve)
  1060. */
  1061. RESET_STORED_API_REQUESTS() {
  1062. return new Promise((resolve, reject) => {
  1063. async.waterfall(
  1064. [
  1065. next => {
  1066. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 1)` });
  1067. YouTubeModule.youtubeApiRequestModel.find({}, next);
  1068. },
  1069. (apiRequests, next) => {
  1070. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 2)` });
  1071. YouTubeModule.youtubeApiRequestModel.deleteMany({}, err => {
  1072. if (err) next("Couldn't reset stored YouTube API requests.");
  1073. else {
  1074. next(null, apiRequests);
  1075. }
  1076. });
  1077. },
  1078. (apiRequests, next) => {
  1079. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 3)` });
  1080. CacheModule.runJob("DEL", { key: "youtubeApiRequestParams" }, this)
  1081. .then(() => next(null, apiRequests))
  1082. .catch(err => next(err));
  1083. },
  1084. (apiRequests, next) => {
  1085. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 4)` });
  1086. CacheModule.runJob("DEL", { key: "youtubeApiRequestResults" }, this)
  1087. .then(() => next(null, apiRequests))
  1088. .catch(err => next(err));
  1089. },
  1090. (apiRequests, next) => {
  1091. this.publishProgress({ status: "update", message: `Resetting stored API requests (stage 5)` });
  1092. async.eachLimit(
  1093. apiRequests.map(apiRequest => apiRequest._id),
  1094. 1,
  1095. (requestId, next) => {
  1096. CacheModule.runJob(
  1097. "PUB",
  1098. {
  1099. channel: "youtube.removeYoutubeApiRequest",
  1100. value: requestId
  1101. },
  1102. this
  1103. )
  1104. .then(() => {
  1105. next();
  1106. })
  1107. .catch(err => {
  1108. next(err);
  1109. });
  1110. },
  1111. err => {
  1112. if (err) next(err);
  1113. else next();
  1114. }
  1115. );
  1116. }
  1117. ],
  1118. err => {
  1119. if (err) reject(new Error(err));
  1120. else resolve();
  1121. }
  1122. );
  1123. });
  1124. }
  1125. /**
  1126. * Remove a stored api request
  1127. *
  1128. * @param {object} payload - object that contains the payload
  1129. * @param {object} payload.requestId - the api request id
  1130. * @returns {Promise} - returns promise (reject, resolve)
  1131. */
  1132. REMOVE_STORED_API_REQUEST(payload) {
  1133. return new Promise((resolve, reject) => {
  1134. async.waterfall(
  1135. [
  1136. next => {
  1137. YouTubeModule.youtubeApiRequestModel.deleteOne({ _id: payload.requestId }, err => {
  1138. if (err) next("Couldn't remove stored YouTube API request.");
  1139. else {
  1140. next();
  1141. }
  1142. });
  1143. },
  1144. next => {
  1145. CacheModule.runJob(
  1146. "HDEL",
  1147. {
  1148. table: "youtubeApiRequestParams",
  1149. key: payload.requestId.toString()
  1150. },
  1151. this
  1152. )
  1153. .then(next)
  1154. .catch(err => next(err));
  1155. },
  1156. next => {
  1157. CacheModule.runJob(
  1158. "HDEL",
  1159. {
  1160. table: "youtubeApiRequestResults",
  1161. key: payload.requestId.toString()
  1162. },
  1163. this
  1164. )
  1165. .then(next)
  1166. .catch(err => next(err));
  1167. },
  1168. next => {
  1169. CacheModule.runJob("PUB", {
  1170. channel: "youtube.removeYoutubeApiRequest",
  1171. value: payload.requestId.toString()
  1172. })
  1173. .then(next)
  1174. .catch(err => next(err));
  1175. }
  1176. ],
  1177. err => {
  1178. if (err) reject(new Error(err));
  1179. else resolve();
  1180. }
  1181. );
  1182. });
  1183. }
  1184. /**
  1185. * Create YouTube videos
  1186. *
  1187. * @param {object} payload - an object containing the payload
  1188. * @param {Array | object} payload.youtubeVideos - the youtubeVideo object or array of
  1189. * @returns {Promise} - returns a promise (resolve, reject)
  1190. */
  1191. CREATE_VIDEOS(payload) {
  1192. return new Promise((resolve, reject) => {
  1193. async.waterfall(
  1194. [
  1195. next => {
  1196. let { youtubeVideos } = payload;
  1197. if (typeof youtubeVideos !== "object") next("Invalid youtubeVideos type");
  1198. else {
  1199. if (!Array.isArray(youtubeVideos)) youtubeVideos = [youtubeVideos];
  1200. YouTubeModule.youtubeVideoModel.insertMany(youtubeVideos, next);
  1201. }
  1202. },
  1203. (youtubeVideos, next) => {
  1204. // TODO support spotify here
  1205. const mediaSources = youtubeVideos.map(video => `youtube:${video.youtubeId}`);
  1206. async.eachLimit(
  1207. mediaSources,
  1208. 2,
  1209. (mediaSource, next) => {
  1210. MediaModule.runJob("RECALCULATE_RATINGS", { mediaSource }, this)
  1211. .then(() => next())
  1212. .catch(next);
  1213. },
  1214. err => {
  1215. if (err) next(err);
  1216. else next(null, youtubeVideos);
  1217. }
  1218. );
  1219. }
  1220. ],
  1221. (err, youtubeVideos) => {
  1222. if (err) reject(new Error(err));
  1223. else resolve({ youtubeVideos });
  1224. }
  1225. );
  1226. });
  1227. }
  1228. /**
  1229. * Get YouTube videos
  1230. *
  1231. * @param {object} payload - an object containing the payload
  1232. * @param {Array} payload.identifiers - an array of YouTube video ObjectId's or YouTube ID's
  1233. * @param {boolean} payload.createMissing - attempt to fetch and create video's if not in db
  1234. * @param {boolean} payload.replaceExisting - replace existing
  1235. * @returns {Promise} - returns a promise (resolve, reject)
  1236. */
  1237. async GET_VIDEOS(payload) {
  1238. const getVideosFromYoutubeIds = async youtubeIds => {
  1239. const jobsToRun = [];
  1240. const chunkSize = 50;
  1241. while (youtubeIds.length > 0) {
  1242. const chunkedYoutubeIds = youtubeIds.splice(0, chunkSize);
  1243. const params = {
  1244. part: "snippet,contentDetails,statistics,status",
  1245. id: chunkedYoutubeIds.join(",")
  1246. };
  1247. jobsToRun.push(YouTubeModule.runJob("API_GET_VIDEOS", { params }, this));
  1248. }
  1249. const jobResponses = await Promise.all(jobsToRun);
  1250. return jobResponses
  1251. .map(jobResponse => jobResponse.response.data.items)
  1252. .flat()
  1253. .map(item => {
  1254. // TODO Clean up duration converter
  1255. let dur = item.contentDetails.duration;
  1256. dur = dur.replace("PT", "");
  1257. let duration = 0;
  1258. dur = dur.replace(/([\d]*)H/, (v, v2) => {
  1259. v2 = Number(v2);
  1260. duration = v2 * 60 * 60;
  1261. return "";
  1262. });
  1263. dur = dur.replace(/([\d]*)M/, (v, v2) => {
  1264. v2 = Number(v2);
  1265. duration += v2 * 60;
  1266. return "";
  1267. });
  1268. dur.replace(/([\d]*)S/, (v, v2) => {
  1269. v2 = Number(v2);
  1270. duration += v2;
  1271. return "";
  1272. });
  1273. const youtubeVideo = {
  1274. youtubeId: item.id,
  1275. title: item.snippet.title,
  1276. author: item.snippet.channelTitle,
  1277. thumbnail: item.snippet.thumbnails.default.url,
  1278. duration,
  1279. uploadedAt: new Date(item.snippet.publishedAt),
  1280. rawData: item
  1281. };
  1282. return youtubeVideo;
  1283. });
  1284. };
  1285. const { identifiers, createMissing, replaceExisting } = payload;
  1286. const youtubeIds = identifiers.filter(identifier => !mongoose.Types.ObjectId.isValid(identifier));
  1287. const objectIds = identifiers.filter(identifier => mongoose.Types.ObjectId.isValid(identifier));
  1288. const existingVideos = (await YouTubeModule.youtubeVideoModel.find({ youtubeId: youtubeIds }))
  1289. .concat(await YouTubeModule.youtubeVideoModel.find({ _id: objectIds }))
  1290. .map(video => video._doc);
  1291. const existingYoutubeIds = existingVideos.map(existingVideo => existingVideo.youtubeId);
  1292. // const existingYoutubeObjectIds = existingVideos.map(existingVideo => existingVideo._id.toString());
  1293. if (!replaceExisting) {
  1294. if (!createMissing) return { videos: existingVideos };
  1295. if (identifiers.length === existingVideos.length || youtubeIds.length === 0)
  1296. return { videos: existingVideos };
  1297. const missingYoutubeIds = youtubeIds.filter(youtubeId => existingYoutubeIds.indexOf(youtubeId) === -1);
  1298. if (missingYoutubeIds.length === 0) return { videos: existingVideos };
  1299. const newVideos = await getVideosFromYoutubeIds(missingYoutubeIds);
  1300. await YouTubeModule.runJob("CREATE_VIDEOS", { youtubeVideos: newVideos }, this);
  1301. return { videos: existingVideos.concat(newVideos) };
  1302. }
  1303. const newVideos = await getVideosFromYoutubeIds(existingYoutubeIds);
  1304. const promises = newVideos.map(newVideo =>
  1305. YouTubeModule.youtubeVideoModel.updateOne(
  1306. { youtubeId: newVideo.youtubeId },
  1307. { $set: { ...newVideo, updatedAt: Date.now(), documentVersion: 2 } }
  1308. )
  1309. );
  1310. await Promise.allSettled(promises);
  1311. return { videos: newVideos };
  1312. }
  1313. /**
  1314. * Get YouTube channels
  1315. *
  1316. * @param {object} payload - an object containing the payload
  1317. * @param {Array} payload.channelIds - an array of YouTube channel id's
  1318. * @returns {Promise} - returns a promise (resolve, reject)
  1319. */
  1320. async GET_CHANNELS_FROM_IDS(payload) {
  1321. const getChannels = async channelIds => {
  1322. const jobsToRun = [];
  1323. const chunkSize = 50;
  1324. while (channelIds.length > 0) {
  1325. const chunkedChannelIds = channelIds.splice(0, chunkSize);
  1326. const params = {
  1327. part: [
  1328. "brandingSettings",
  1329. "contentDetails",
  1330. "contentOwnerDetails",
  1331. "id",
  1332. "localizations",
  1333. "snippet",
  1334. "statistics",
  1335. "status",
  1336. "topicDetails"
  1337. ].join(","),
  1338. id: chunkedChannelIds.join(",")
  1339. };
  1340. jobsToRun.push(YouTubeModule.runJob("API_GET_CHANNELS", { params }, this));
  1341. }
  1342. const jobResponses = await Promise.all(jobsToRun);
  1343. return jobResponses
  1344. .map(jobResponse => jobResponse.response.data.items)
  1345. .flat()
  1346. .map(item => {
  1347. const youtubeChannel = {
  1348. channelId: item.id,
  1349. title: item.snippet.title,
  1350. customUrl: item.snippet.customUrl,
  1351. rawData: item
  1352. };
  1353. return youtubeChannel;
  1354. });
  1355. };
  1356. const { channelIds } = payload;
  1357. const existingChannels = (await YouTubeModule.youtubeChannelModel.find({ channelId: channelIds })).map(
  1358. channel => channel._doc
  1359. );
  1360. const existingChannelIds = existingChannels.map(existingChannel => existingChannel.channelId);
  1361. // const existingChannelObjectIds = existingChannels.map(existingChannel => existingChannel._id.toString());
  1362. if (channelIds.length === existingChannels.length) return { channels: existingChannels };
  1363. const missingChannelIds = channelIds.filter(channelId => existingChannelIds.indexOf(channelId) === -1);
  1364. if (missingChannelIds.length === 0) return { videos: existingChannels };
  1365. const newChannels = await getChannels(missingChannelIds);
  1366. await YouTubeModule.youtubeChannelModel.insertMany(newChannels);
  1367. return { channels: existingChannels.concat(newChannels) };
  1368. }
  1369. /**
  1370. * Remove YouTube videos
  1371. *
  1372. * @param {object} payload - an object containing the payload
  1373. * @param {string} payload.videoIds - Array of youtubeVideo ObjectIds
  1374. * @returns {Promise} - returns a promise (resolve, reject)
  1375. */
  1376. REMOVE_VIDEOS(payload) {
  1377. return new Promise((resolve, reject) => {
  1378. let { videoIds } = payload;
  1379. if (!Array.isArray(videoIds)) videoIds = [videoIds];
  1380. async.waterfall(
  1381. [
  1382. next => {
  1383. if (!videoIds.every(videoId => mongoose.Types.ObjectId.isValid(videoId)))
  1384. next("One or more videoIds are not a valid ObjectId.");
  1385. else {
  1386. this.publishProgress({ status: "update", message: `Removing video (stage 1)` });
  1387. YouTubeModule.youtubeVideoModel.find({ _id: { $in: videoIds } }, (err, videos) => {
  1388. if (err) next(err);
  1389. else
  1390. next(
  1391. null,
  1392. videos.map(video => video.youtubeId)
  1393. );
  1394. });
  1395. }
  1396. },
  1397. (youtubeIds, next) => {
  1398. this.publishProgress({ status: "update", message: `Removing video (stage 2)` });
  1399. SongsModule.SongModel.find({ youtubeId: { $in: youtubeIds } }, (err, songs) => {
  1400. if (err) next(err);
  1401. else {
  1402. const filteredIds = youtubeIds.filter(
  1403. youtubeId => !songs.find(song => song.youtubeId === youtubeId)
  1404. );
  1405. if (filteredIds.length < youtubeIds.length)
  1406. next("One or more videos are attached to songs.");
  1407. else next(null, filteredIds);
  1408. }
  1409. });
  1410. },
  1411. (youtubeIds, next) => {
  1412. this.publishProgress({ status: "update", message: `Removing video (stage 3)` });
  1413. MediaModule.runJob("REMOVE_RATINGS", { youtubeIds }, this)
  1414. .then(() => next(null, youtubeIds))
  1415. .catch(next);
  1416. },
  1417. (youtubeIds, next) => {
  1418. this.publishProgress({ status: "update", message: `Removing video (stage 4)` });
  1419. async.eachLimit(
  1420. youtubeIds,
  1421. 2,
  1422. (youtubeId, next) => {
  1423. async.waterfall(
  1424. [
  1425. next => {
  1426. PlaylistsModule.playlistModel.find(
  1427. { "songs.youtubeId": youtubeId },
  1428. (err, playlists) => {
  1429. if (err) next(err);
  1430. else {
  1431. async.eachLimit(
  1432. playlists,
  1433. 1,
  1434. (playlist, next) => {
  1435. PlaylistsModule.runJob(
  1436. "REMOVE_FROM_PLAYLIST",
  1437. { playlistId: playlist._id, youtubeId },
  1438. this
  1439. )
  1440. .then(() => next())
  1441. .catch(next);
  1442. },
  1443. next
  1444. );
  1445. }
  1446. }
  1447. );
  1448. },
  1449. next => {
  1450. StationsModule.stationModel.find(
  1451. { "queue.youtubeId": youtubeId },
  1452. (err, stations) => {
  1453. if (err) next(err);
  1454. else {
  1455. async.eachLimit(
  1456. stations,
  1457. 1,
  1458. (station, next) => {
  1459. StationsModule.runJob(
  1460. "REMOVE_FROM_QUEUE",
  1461. { stationId: station._id, youtubeId },
  1462. this
  1463. )
  1464. .then(() => next())
  1465. .catch(err => {
  1466. if (
  1467. err === "Station not found" ||
  1468. err ===
  1469. "Song is not currently in the queue."
  1470. )
  1471. next();
  1472. else next(err);
  1473. });
  1474. },
  1475. next
  1476. );
  1477. }
  1478. }
  1479. );
  1480. },
  1481. next => {
  1482. StationsModule.stationModel.find(
  1483. { "currentSong.youtubeId": youtubeId },
  1484. (err, stations) => {
  1485. if (err) next(err);
  1486. else {
  1487. async.eachLimit(
  1488. stations,
  1489. 1,
  1490. (station, next) => {
  1491. StationsModule.runJob(
  1492. "SKIP_STATION",
  1493. {
  1494. stationId: station._id,
  1495. natural: false,
  1496. skipReason: "other"
  1497. },
  1498. this
  1499. )
  1500. .then(() => {
  1501. next();
  1502. })
  1503. .catch(err => {
  1504. if (err.message === "Station not found.")
  1505. next();
  1506. else next(err);
  1507. });
  1508. },
  1509. next
  1510. );
  1511. }
  1512. }
  1513. );
  1514. }
  1515. ],
  1516. next
  1517. );
  1518. },
  1519. next
  1520. );
  1521. },
  1522. next => {
  1523. this.publishProgress({ status: "update", message: `Removing video (stage 5)` });
  1524. YouTubeModule.youtubeVideoModel.deleteMany({ _id: { $in: videoIds } }, next);
  1525. },
  1526. (res, next) => {
  1527. this.publishProgress({ status: "update", message: `Removing video (stage 6)` });
  1528. CacheModule.runJob("PUB", {
  1529. channel: "youtube.removeVideos",
  1530. value: videoIds
  1531. })
  1532. .then(next)
  1533. .catch(err => next(err));
  1534. }
  1535. ],
  1536. err => {
  1537. if (err) reject(new Error(err));
  1538. else resolve();
  1539. }
  1540. );
  1541. });
  1542. }
  1543. /**
  1544. * Request a set of YouTube videos
  1545. *
  1546. * @param {object} payload - an object containing the payload
  1547. * @param {string} payload.url - the url of the the YouTube playlist or channel
  1548. * @param {boolean} payload.musicOnly - whether to only get music from the playlist/channel
  1549. * @param {boolean} payload.returnVideos - whether to return videos
  1550. * @returns {Promise} - returns a promise (resolve, reject)
  1551. */
  1552. REQUEST_SET(payload) {
  1553. return new Promise((resolve, reject) => {
  1554. async.waterfall(
  1555. [
  1556. next => {
  1557. const playlistRegex = /[\\?&]list=([^&#]*)/;
  1558. const channelRegex =
  1559. /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/;
  1560. if (playlistRegex.exec(payload.url) || channelRegex.exec(payload.url))
  1561. YouTubeModule.runJob(
  1562. playlistRegex.exec(payload.url) ? "GET_PLAYLIST" : "GET_CHANNEL_VIDEOS",
  1563. {
  1564. url: payload.url,
  1565. musicOnly: payload.musicOnly
  1566. },
  1567. this
  1568. )
  1569. .then(res => {
  1570. next(null, res.songs);
  1571. })
  1572. .catch(next);
  1573. else next("Invalid YouTube URL.");
  1574. },
  1575. async youtubeIds => {
  1576. if (youtubeIds.length === 0) return { videos: [] };
  1577. const { videos } = await YouTubeModule.runJob(
  1578. "GET_VIDEOS",
  1579. { identifiers: youtubeIds, createMissing: true },
  1580. this
  1581. );
  1582. return { videos };
  1583. }
  1584. ],
  1585. (err, response) => {
  1586. if (err) reject(new Error(err));
  1587. else resolve(response);
  1588. }
  1589. );
  1590. });
  1591. }
  1592. /**
  1593. * Gets missing YouTube video's from all playlists, stations and songs
  1594. *
  1595. * @returns {Promise} - returns a promise (resolve, reject)
  1596. */
  1597. async GET_MISSING_VIDEOS() {
  1598. const youtubeIds = Array.from(
  1599. new Set(
  1600. [
  1601. ...(await SongsModule.runJob("GET_ALL_MEDIA_SOURCES", {}, this)),
  1602. ...(await PlaylistsModule.runJob("GET_ALL_MEDIA_SOURCES", {}, this))
  1603. ]
  1604. .filter(mediaSource => mediaSource.startsWith("youtube:"))
  1605. .map(mediaSource => mediaSource.split(":")[1])
  1606. )
  1607. );
  1608. const existingYoutubeIds = await YouTubeModule.youtubeVideoModel.distinct("youtubeId");
  1609. const missingYoutubeIds = youtubeIds.filter(youtubeId => existingYoutubeIds.indexOf(youtubeId) === -1);
  1610. const res = await YouTubeModule.runJob(
  1611. "GET_VIDEOS",
  1612. { identifiers: missingYoutubeIds, createMissing: true },
  1613. this
  1614. );
  1615. const gotVideos = res.videos;
  1616. return {
  1617. all: youtubeIds.length,
  1618. existing: existingYoutubeIds.length,
  1619. missing: missingYoutubeIds.length,
  1620. got: gotVideos.length
  1621. };
  1622. }
  1623. /**
  1624. * Updates videos from version 1 to version 2
  1625. *
  1626. * @returns {Promise} - returns a promise (resolve, reject)
  1627. */
  1628. async UPDATE_VIDEOS_V1_TO_V2() {
  1629. const videoIds = await YouTubeModule.youtubeVideoModel.distinct("_id", { documentVersion: 1 });
  1630. const res = await YouTubeModule.runJob("GET_VIDEOS", { identifiers: videoIds, replaceExisting: true }, this);
  1631. const v1 = videoIds.length;
  1632. const v2 = res.videos.length;
  1633. return {
  1634. v1,
  1635. v2
  1636. };
  1637. }
  1638. /**
  1639. * Gets missing YouTube channels based on cached YouTube video's
  1640. *
  1641. * @returns {Promise} - returns a promise (resolve, reject)
  1642. */
  1643. async GET_MISSING_CHANNELS() {
  1644. const currentChannelIds = await YouTubeModule.youtubeChannelModel.distinct("channelId");
  1645. const videoChannelIds = await YouTubeModule.youtubeVideoModel.distinct("rawData.snippet.channelId");
  1646. const missingChannelIds = videoChannelIds.filter(channelId => currentChannelIds.indexOf(channelId) === -1);
  1647. const res = await YouTubeModule.runJob("GET_CHANNELS_FROM_IDS", { channelIds: missingChannelIds }, this);
  1648. const gotChannels = res.channels;
  1649. return {
  1650. current: currentChannelIds.length,
  1651. all: videoChannelIds.length,
  1652. missing: missingChannelIds.length,
  1653. got: gotChannels.length
  1654. };
  1655. }
  1656. }
  1657. export default new _YouTubeModule();