/* eslint-disable */ import async from "async"; import config from "config"; import * as rax from "retry-axios"; import axios from "axios"; import CoreClass from "../core"; class RateLimitter { /** * Constructor * * @param {number} timeBetween - The time between each allowed YouTube request */ constructor(timeBetween) { this.dateStarted = Date.now(); this.timeBetween = timeBetween; } /** * Returns a promise that resolves whenever the ratelimit of a YouTube request is done * * @returns {Promise} - promise that gets resolved when the rate limit allows it */ continue() { return new Promise(resolve => { if (Date.now() - this.dateStarted >= this.timeBetween) resolve(); else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now()); }); } /** * Restart the rate limit timer */ restart() { this.dateStarted = Date.now(); } } let YouTubeModule; const quotas = [ { type: "QUERIES_PER_DAY", limit: 10000 }, { type: "QUERIES_PER_MINUTE", limit: 1800000 }, { type: "QUERIES_PER_100_SECONDS", limit: 3000000 } ]; // const dummyApiCalls = [ // { // quotaCost: 100, // date: new Date(new Date() - (1000 * 120)) // }, // { // quotaCost: 2, // date: new Date(new Date() - (1000 * 120)) // }, // { // quotaCost: 1, // date: new Date() // }, // { // quotaCost: 100, // date: new Date() // } // ]; const isQuotaExceeded = apiCalls => { const reversedApiCalls = apiCalls.slice().reverse(); const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit); let quotaExceeded = false; for (const quota of sortedQuotas) { let quotaUsed = 0; let dateCutoff = null; if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date() - 1000 * 60; else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date() - 1000 * 100; else if (quota.type === "QUERIES_PER_DAY") { // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT dateCutoff = new Date(); dateCutoff.setUTCMilliseconds(0); dateCutoff.setUTCSeconds(0); dateCutoff.setUTCMinutes(0); dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7); dateCutoff.setUTCHours(0); } for (const apiCall of reversedApiCalls) { if (apiCall.date >= dateCutoff) quotaUsed += apiCall.quotaCost; else break; } if (quotaUsed >= quota.limit) { quotaExceeded = true; break; } } return quotaExceeded; }; const getQuotaStatus = apiCalls => { const reversedApiCalls = apiCalls.slice().reverse(); const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit); const status = {}; for (const quota of sortedQuotas) { status[quota.type] = { quotaUsed: 0, limit: quota.limit, quotaExceeded: false }; let dateCutoff = null; if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date() - 1000 * 60; else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date() - 1000 * 100; else if (quota.type === "QUERIES_PER_DAY") { // Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT dateCutoff = new Date(); dateCutoff.setUTCMilliseconds(0); dateCutoff.setUTCSeconds(0); dateCutoff.setUTCMinutes(0); dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7); dateCutoff.setUTCHours(0); } for (const apiCall of reversedApiCalls) { if (apiCall.date >= dateCutoff) status[quota.type].quotaUsed += apiCall.quotaCost; else break; } if (status[quota.type].quotaUsed >= quota.limit && !status[quota.type].quotaExceeded) status[quota.type].quotaExceeded = true; } return status; }; class _YouTubeModule extends CoreClass { // eslint-disable-next-line require-jsdoc constructor() { super("youtube", { concurrency: 1, priorities: { GET_PLAYLIST: 11 } }); YouTubeModule = this; } /** * Initialises the activities module * * @returns {Promise} - returns promise (reject, resolve) */ initialize() { return new Promise(resolve => { this.rateLimiter = new RateLimitter(config.get("apis.youtube.rateLimit")); this.requestTimeout = config.get("apis.youtube.requestTimeout"); this.axios = axios.create(); this.axios.defaults.raxConfig = { instance: this.axios, retry: config.get("apis.youtube.retryAmount"), noResponseRetries: config.get("apis.youtube.retryAmount") }; rax.attach(this.axios); this.apiCalls = []; resolve(); }); } /** * Fetches a list of songs from Youtube's API * * @param {object} payload - object that contains the payload * @param {string} payload.query - the query we'll pass to youtubes api * @param {string} payload.pageToken - (optional) if this exists, will search search youtube for a specific page reference * @returns {Promise} - returns promise (reject, resolve) */ SEARCH(payload) { const params = { part: "snippet", q: payload.query, type: "video", maxResults: 10 }; if (payload.pageToken) params.pageToken = payload.pageToken; return new Promise((resolve, reject) => { YouTubeModule.runJob( "API_SEARCH", { params }, this ) .then(({ response }) => { const { data } = response; return resolve(data); }) .catch(err => { YouTubeModule.log("ERROR", "SEARCH", `${err.message}`); return reject(new Error("An error has occured. Please try again later.")); }); }); } /** * Gets the details of a song using the YouTube API * * @param {object} payload - object that contains the payload * @param {string} payload.youtubeId - the YouTube API id of the song * @returns {Promise} - returns promise (reject, resolve) */ GET_SONG(payload) { return new Promise((resolve, reject) => { const params = { part: "snippet,contentDetails,statistics,status", id: payload.youtubeId }; YouTubeModule.runJob("API_GET_VIDEOS", { params }, this) .then(({ response }) => { const { data } = response; if (data.items[0] === undefined) return reject(new Error("The specified video does not exist or cannot be publicly accessed.")); // TODO Clean up duration converter let dur = data.items[0].contentDetails.duration; dur = dur.replace("PT", ""); let duration = 0; dur = dur.replace(/([\d]*)H/, (v, v2) => { v2 = Number(v2); duration = v2 * 60 * 60; return ""; }); dur = dur.replace(/([\d]*)M/, (v, v2) => { v2 = Number(v2); duration += v2 * 60; return ""; }); // eslint-disable-next-line no-unused-vars dur = dur.replace(/([\d]*)S/, (v, v2) => { v2 = Number(v2); duration += v2; return ""; }); const song = { youtubeId: data.items[0].id, title: data.items[0].snippet.title, thumbnail: data.items[0].snippet.thumbnails.default.url, duration }; return resolve({ song }); }) .catch(err => { YouTubeModule.log("ERROR", "GET_SONG", `${err.message}`); return reject(new Error("An error has occured. Please try again later.")); }); }); } /** * Gets the id of the channel upload playlist * * @param {object} payload - object that contains the payload * @param {string} payload.id - the id of the YouTube channel. Optional: can be left out if specifying a username. * @param {string} payload.username - the username of the YouTube channel. Only gets used if no id is specified. * @returns {Promise} - returns promise (reject, resolve) */ GET_CHANNEL_UPLOADS_PLAYLIST_ID(payload) { return new Promise((resolve, reject) => { const params = { part: "id,contentDetails" }; if (payload.id) params.id = payload.id; else params.forUsername = payload.username; YouTubeModule.runJob( "API_GET_CHANNELS", { params }, this ) .then(({ response }) => { const { data } = response; if (data.pageInfo.totalResults === 0) return reject(new Error("Channel not found.")); const playlistId = data.items[0].contentDetails.relatedPlaylists.uploads; return resolve({ playlistId }); }) .catch(err => { YouTubeModule.log("ERROR", "GET_CHANNEL_UPLOADS_PLAYLIST_ID", `${err.message}`); if (err.message === "Request failed with status code 404") { return reject(new Error("Channel not found. Is the channel public/unlisted?")); } return reject(new Error("An error has occured. Please try again later.")); }); }); } /** * Gets the id of the channel from the custom URL * * @param {object} payload - object that contains the payload * @param {string} payload.customUrl - the customUrl of the YouTube channel * @returns {Promise} - returns promise (reject, resolve) */ GET_CHANNEL_ID_FROM_CUSTOM_URL(payload) { return new Promise((resolve, reject) => { async.waterfall( [ next => { const params = { part: "snippet", type: "channel", maxResults: 50 }; params.q = payload.customUrl; YouTubeModule.runJob( "API_SEARCH", { params }, this ) .then(({ response }) => { const { data } = response; if (data.pageInfo.totalResults === 0) return next("Channel not found."); const channelIds = data.items.map(item => item.id.channelId); return next(null, channelIds); }) .catch(err => { next(err); }); }, (channelIds, next) => { const params = { part: "snippet", id: channelIds.join(","), maxResults: 50 }; YouTubeModule.runJob( "API_GET_CHANNELS", { params }, this ) .then(({ response }) => { const { data } = response; if (data.pageInfo.totalResults === 0) return next("Channel not found."); let channelId = null; for (const item of data.items) { if (item.snippet.customUrl && item.snippet.customUrl.toLowerCase() === payload.customUrl.toLowerCase()) { channelId = item.id; break; } } if (!channelId) return next("Channel not found."); return next(null, channelId); }) .catch(err => { next(err); }); } ], (channelId, err) => { if (err) { YouTubeModule.log("ERROR", "GET_CHANNEL_ID_FROM_CUSTOM_URL", `${err.message}`); if (err.message === "Request failed with status code 404") { return reject(new Error("Channel not found. Is the channel public/unlisted?")); } return reject(new Error("An error has occured. Please try again later.")); } return resolve({ channelId }); } ); }); } /** * Returns an array of songs taken from a YouTube playlist * * @param {object} payload - object that contains the payload * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the playlist * @param {string} payload.url - the url of the YouTube playlist * @returns {Promise} - returns promise (reject, resolve) */ GET_PLAYLIST(payload) { return new Promise((resolve, reject) => { const regex = /[\\?&]list=([^&#]*)/; const splitQuery = regex.exec(payload.url); if (!splitQuery) { YouTubeModule.log("ERROR", "GET_PLAYLIST", "Invalid YouTube playlist URL query."); reject(new Error("Invalid playlist URL.")); return; } const playlistId = splitQuery[1]; async.waterfall( [ next => { let songs = []; let nextPageToken = ""; async.whilst( next => { YouTubeModule.log( "INFO", `Getting playlist progress for job (${this.toString()}): ${ songs.length } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.` ); next(null, nextPageToken !== undefined); }, next => { // Add 250ms delay between each job request setTimeout(() => { YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this) .then(response => { songs = songs.concat(response.songs); nextPageToken = response.nextPageToken; next(); }) .catch(err => next(err)); }, 250); }, err => next(err, songs) ); }, (songs, next) => next( null, songs.map(song => song.contentDetails.videoId) ), (songs, next) => { if (!payload.musicOnly) return next(true, { songs }); return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this) .then(filteredSongs => next(null, { filteredSongs, songs })) .catch(next); } ], (err, response) => { if (err && err !== true) { YouTubeModule.log("ERROR", "GET_PLAYLIST", "Some error has occurred.", err.message); reject(new Error(err.message)); } else { resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs }); } } ); }); } /** * Returns a a page from a YouTube playlist. Is used internally by GET_PLAYLIST and GET_CHANNEL. * * @param {object} payload - object that contains the payload * @param {boolean} payload.playlistId - the playlist id to get videos from * @param {boolean} payload.nextPageToken - the nextPageToken to use * @param {string} payload.url - the url of the YouTube playlist * @returns {Promise} - returns promise (reject, resolve) */ GET_PLAYLIST_PAGE(payload) { return new Promise((resolve, reject) => { const params = { part: "contentDetails", playlistId: payload.playlistId, maxResults: 50 }; if (payload.nextPageToken) params.pageToken = payload.nextPageToken; YouTubeModule.runJob( "GET_PLAYLIST_ITEMS", { params }, this ) .then(({ response }) => { const { data } = response; const songs = data.items; if (data.nextPageToken) return resolve({ nextPageToken: data.nextPageToken, songs }); return resolve({ songs }); }) .catch(err => { YouTubeModule.log("ERROR", "GET_PLAYLIST_PAGE", `${err.message}`); if (err.message === "Request failed with status code 404") { return reject(new Error("Playlist not found. Is the playlist public/unlisted?")); } return reject(new Error("An error has occured. Please try again later.")); }); }); } /** * Filters a list of YouTube videos so that they only contains videos with music. Is used internally by GET_PLAYLIST * * @param {object} payload - object that contains the payload * @param {Array} payload.videoIds - an array of YouTube videoIds to filter through * @param {Array} payload.page - the current page/set of video's to get, starting at 0. If left null, 0 is assumed. Will recurse. * @returns {Promise} - returns promise (reject, resolve) */ FILTER_MUSIC_VIDEOS(payload) { return new Promise((resolve, reject) => { const page = payload.page ? payload.page : 0; const videosPerPage = 50; const localVideoIds = payload.videoIds.splice(page * 50, videosPerPage); if (localVideoIds.length === 0) { resolve({ videoIds: [] }); return; } const params = { part: "topicDetails", id: localVideoIds.join(","), maxResults: videosPerPage }; YouTubeModule.runJob("API_GET_VIDEOS", { params }, this) .then(({ response }) => { const { data } = response; const videoIds = []; data.items.forEach(item => { const videoId = item.id; if (!item.topicDetails) return; if (item.topicDetails.topicCategories.indexOf("https://en.wikipedia.org/wiki/Music") !== -1) videoIds.push(videoId); }); return YouTubeModule.runJob( "FILTER_MUSIC_VIDEOS", { videoIds: payload.videoIds, page: page + 1 }, this ) .then(result => resolve({ videoIds: videoIds.concat(result.videoIds) })) .catch(err => reject(err)); }) .catch(err => { YouTubeModule.log("ERROR", "FILTER_MUSIC_VIDEOS", `${err.message}`); return reject(new Error("Failed to find playlist from YouTube")); }); }); } /** * Returns an array of songs taken from a YouTube channel * * @param {object} payload - object that contains the payload * @param {boolean} payload.musicOnly - whether to return music videos or all videos in the channel * @param {string} payload.url - the url of the YouTube channel * @returns {Promise} - returns promise (reject, resolve) */ GET_CHANNEL(payload) { return new Promise((resolve, reject) => { const regex = /\.[\w]+\/(?:(?:channel\/(UC[0-9A-Za-z_-]{21}[AQgw]))|(?:user\/?([\w-]+))|(?:c\/?([\w-]+))|(?:\/?([\w-]+)))/; const splitQuery = regex.exec(payload.url); if (!splitQuery) { YouTubeModule.log("ERROR", "GET_CHANNEL", "Invalid YouTube channel URL query."); reject(new Error("Invalid playlist URL.")); return; } const channelId = splitQuery[1]; const channelUsername = splitQuery[2]; const channelCustomUrl = splitQuery[3]; // NOTE: not supported yet const channelUsernameOrCustomUrl = splitQuery[4]; // NOTE: customUrl not supported yet console.log(`Channel id: ${channelId}`); console.log(`Channel username: ${channelUsername}`); console.log(`Channel custom URL: ${channelCustomUrl}`); console.log(`Channel username or custom URL: ${channelUsernameOrCustomUrl}`); async.waterfall( [ next => { const payload = {}; if (channelId) payload.id = channelId; else if (channelUsername || channelUsernameOrCustomUrl) payload.username = channelUsername; else return next("No id/username given."); return YouTubeModule.runJob("GET_CHANNEL_UPLOADS_PLAYLIST_ID", payload, this) .then(({ playlistId }) => { next(null, playlistId); }) .catch(err => next(err)); }, next => { let songs = []; let nextPageToken = ""; async.whilst( next => { YouTubeModule.log( "INFO", `Getting channel progress for job (${this.toString()}): ${ songs.length } songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.` ); next(null, nextPageToken !== undefined); }, next => { // Add 250ms delay between each job request setTimeout(() => { YouTubeModule.runJob("GET_PLAYLIST_PAGE", { playlistId, nextPageToken }, this) .then(response => { songs = songs.concat(response.songs); nextPageToken = response.nextPageToken; next(); }) .catch(err => next(err)); }, 250); }, err => next(err, songs) ); }, (songs, next) => next( null, songs.map(song => song.contentDetails.videoId) ), (songs, next) => { if (!payload.musicOnly) return next(true, { songs }); return YouTubeModule.runJob("FILTER_MUSIC_VIDEOS", { videoIds: songs.slice() }, this) .then(filteredSongs => next(null, { filteredSongs, songs })) .catch(next); } ], (err, response) => { if (err && err !== true) { YouTubeModule.log("ERROR", "GET_CHANNEL", "Some error has occurred.", err.message); reject(new Error(err.message)); } else { resolve({ songs: response.filteredSongs ? response.filteredSongs.videoIds : response.songs }); } } ); }); } API_GET_VIDEOS(payload) { return new Promise((resolve, reject) => { const { params } = payload; YouTubeModule.runJob( "API_CALL", { url: "https://www.googleapis.com/youtube/v3/videos", params: { key: config.get("apis.youtube.key"), ...params }, quotaCost: 1 }, this ) .then(response => { resolve(response); }) .catch(err => { reject(err); }); }); } API_GET_PLAYLIST_ITEMS(payload) { return new Promise((resolve, reject) => { const { params } = payload; YouTubeModule.runJob( "API_CALL", { url: "https://www.googleapis.com/youtube/v3/playlistItems", params: { key: config.get("apis.youtube.key"), ...params }, quotaCost: 1 }, this ) .then(response => { resolve(response); }) .catch(err => { reject(err); }); }); } API_GET_CHANNELS(payload) { return new Promise((resolve, reject) => { const { params } = payload; YouTubeModule.runJob( "API_CALL", { url: "https://www.googleapis.com/youtube/v3/channels", params: { key: config.get("apis.youtube.key"), ...params }, quotaCost: 1 }, this ) .then(response => { resolve(response); }) .catch(err => { reject(err); }); }); } API_SEARCH(payload) { return new Promise((resolve, reject) => { const { params } = payload; YouTubeModule.runJob( "API_CALL", { url: "https://www.googleapis.com/youtube/v3/search", params: { key: config.get("apis.youtube.key"), ...params }, quotaCost: 100 }, this ) .then(response => { resolve(response); }) .catch(err => { reject(err); }); }); } API_CALL(payload) { return new Promise((resolve, reject) => { const { url, params, quotaCost } = payload; const quotaExceeded = isQuotaExceeded(YouTubeModule.apiCalls); if (quotaExceeded) reject(new Error("Quota has been exceeded. Please wait a while.")); else { YouTubeModule.apiCalls.push({ quotaCost, date: new Date() }); YouTubeModule.axios .get(url, { params, timeout: YouTubeModule.requestTimeout }) .then(response => { if (response.data.error) { reject(new Error(response.data.error)); } else { resolve({ response }); } }) .catch(err => { reject(err); }); } }); } } export default new _YouTubeModule();