feat: worked on new YouTube admin module and jobs/actions for getting api requests and quota status

backend/logic/actions/index.js

@@ -9,6 +9,7 @@ import reports from "./reports";
 import news from "./news";
 import punishments from "./punishments";
 import utils from "./utils";
+import youtube from "./youtube";
 
 export default {
 	apis,
@@ -21,5 +22,6 @@ export default {
 	reports,
 	news,
 	punishments,
-	utils
+	utils,
+	youtube
 };

backend/logic/actions/youtube.js

@@ -0,0 +1,71 @@
+import { isAdminRequired } from "./hooks";
+
+// eslint-disable-next-line
+import moduleManager from "../../index";
+
+const UtilsModule = moduleManager.modules.utils;
+const YouTubeModule = moduleManager.modules.youtube;
+
+export default {
+	/**
+	 * Returns details about the YouTube quota usage
+	 *
+	 * @returns {{status: string, data: object}}
+	 */
+	getQuotaStatus: isAdminRequired(function getQuotaStatus(session, fromDate, cb) {
+		YouTubeModule.runJob("GET_QUOTA_STATUS", { fromDate }, this)
+			.then(response => {
+				this.log("SUCCESS", "YOUTUBE_GET_QUOTA_STATUS", `Getting quota status was successful.`);
+				return cb({ status: "success", data: { status: response.status } });
+			})
+			.catch(async err => {
+				err = await UtilsModule.runJob("GET_ERROR", { error: err }, this);
+				this.log("ERROR", "YOUTUBE_GET_QUOTA_STATUS", `Getting quota status failed. "${err}"`);
+				return cb({ status: "error", message: err });
+			});
+	}),
+
+	/**
+	 * Returns api requests
+	 *
+	 * @returns {{status: string, data: object}}
+	 */
+	getApiRequests: isAdminRequired(function getApiRequests(session, fromDate, cb) {
+		YouTubeModule.runJob("GET_API_REQUESTS", { fromDate }, this)
+			.then(response => {
+				this.log("SUCCESS", "YOUTUBE_GET_API_REQUESTS", `Getting api requests was successful.`);
+				return cb({ status: "success", data: { apiRequests: response.apiRequests } });
+			})
+			.catch(async err => {
+				err = await UtilsModule.runJob("GET_ERROR", { error: err }, this);
+				this.log("ERROR", "YOUTUBE_GET_API_REQUESTS", `Getting api requests failed. "${err}"`);
+				return cb({ status: "error", message: err });
+			});
+	}),
+
+	/**
+	 * Returns a specific api request
+	 *
+	 * @returns {{status: string, data: object}}
+	 */
+	getApiRequest: isAdminRequired(function getApiRequest(session, apiRequestId, cb) {
+		YouTubeModule.runJob("GET_API_REQUEST", { apiRequestId }, this)
+			.then(response => {
+				this.log(
+					"SUCCESS",
+					"YOUTUBE_GET_API_REQUEST",
+					`Getting api request with id ${apiRequestId} was successful.`
+				);
+				return cb({ status: "success", data: { apiRequest: response.apiRequest } });
+			})
+			.catch(async err => {
+				err = await UtilsModule.runJob("GET_ERROR", { error: err }, this);
+				this.log(
+					"ERROR",
+					"YOUTUBE_GET_API_REQUEST",
+					`Getting api request with id ${apiRequestId} failed. "${err}"`
+				);
+				return cb({ status: "error", message: err });
+			});
+	})
+};

backend/logic/youtube.js

@@ -133,7 +133,7 @@ class _YouTubeModule extends CoreClass {
 			rax.attach(this.axios);
 
 			this.youtubeApiRequestModel
-				.find({ $gte: new Date() - 2 * 24 * 60 * 60 * 1000 }, { date: true, quotaCost: true, _id: false })
+				.find({ date: { $gte: new Date() - 2 * 24 * 60 * 60 * 1000 } }, { date: true, quotaCost: true, _id: false })
 				.sort({ date: 1 })
 				.exec((err, youtubeApiRequests) => {
 					if (err) console.log("Couldn't load YouTube API requests.");
@@ -183,42 +183,52 @@ class _YouTubeModule extends CoreClass {
 		});
 	}
 
-	GET_QUOTA_STATUS() {
-		return new Promise(resolve => {
-			const reversedApiCalls = YouTubeModule.apiCalls.slice().reverse();
-			const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
-			const status = {};
-
-			for (const quota of sortedQuotas) {
-				status[quota.type] = {
-					quotaUsed: 0,
-					limit: quota.limit,
-					quotaExceeded: false
-				};
-				let dateCutoff = null;
-
-				if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date() - 1000 * 60;
-				else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date() - 1000 * 100;
-				else if (quota.type === "QUERIES_PER_DAY") {
-					// Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
-					dateCutoff = new Date();
-					dateCutoff.setUTCMilliseconds(0);
-					dateCutoff.setUTCSeconds(0);
-					dateCutoff.setUTCMinutes(0);
-					dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
-					dateCutoff.setUTCHours(0);
-				}
-
-				for (const apiCall of reversedApiCalls) {
-					if (apiCall.date >= dateCutoff) status[quota.type].quotaUsed += apiCall.quotaCost;
-					else break;
-				}
+	GET_QUOTA_STATUS(payload) {
+		return new Promise((resolve, reject) => {
+			const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
 
-				if (status[quota.type].quotaUsed >= quota.limit && !status[quota.type].quotaExceeded)
-					status[quota.type].quotaExceeded = true;
-			}
+			YouTubeModule.youtubeApiRequestModel
+				.find({ date: { $gte: fromDate - 2 * 24 * 60 * 60 * 1000, $lte: fromDate } }, { date: true, quotaCost: true, _id: false })
+				.sort({ date: 1 })
+				.exec((err, youtubeApiRequests) => {
+					if (err) reject(new Error("Couldn't load YouTube API requests."));
+					else {
+						const reversedApiCalls = youtubeApiRequests.slice().reverse();
+						const sortedQuotas = quotas.sort((a, b) => a.limit > b.limit);
+						const status = {};
+
+						for (const quota of sortedQuotas) {
+							status[quota.type] = {
+								quotaUsed: 0,
+								limit: quota.limit,
+								quotaExceeded: false
+							};
+							let dateCutoff = null;
+
+							if (quota.type === "QUERIES_PER_MINUTE") dateCutoff = new Date(fromDate) - 1000 * 60;
+							else if (quota.type === "QUERIES_PER_100_SECONDS") dateCutoff = new Date(fromDate) - 1000 * 100;
+							else if (quota.type === "QUERIES_PER_DAY") {
+								// Quota resets at midnight PT, this is my best guess to convert the current date to the last midnight PT
+								dateCutoff = new Date(fromDate);
+								dateCutoff.setUTCMilliseconds(0);
+								dateCutoff.setUTCSeconds(0);
+								dateCutoff.setUTCMinutes(0);
+								dateCutoff.setUTCHours(dateCutoff.getUTCHours() - 7);
+								dateCutoff.setUTCHours(0);
+							}
+
+							for (const apiCall of reversedApiCalls) {
+								if (apiCall.date >= dateCutoff) status[quota.type].quotaUsed += apiCall.quotaCost;
+								else break;
+							}
+
+							if (status[quota.type].quotaUsed >= quota.limit && !status[quota.type].quotaExceeded)
+								status[quota.type].quotaExceeded = true;
+						}
 
-			resolve({ status });
+						resolve({ status });
+					}
+				});
 		});
 	}
 
@@ -452,8 +462,7 @@ class _YouTubeModule extends CoreClass {
 							next => {
 								YouTubeModule.log(
 									"INFO",
-									`Getting playlist progress for job (${this.toString()}): ${
-										songs.length
+									`Getting playlist progress for job (${this.toString()}): ${songs.length
 									} songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
 								);
 								next(null, nextPageToken !== undefined);
@@ -652,8 +661,7 @@ class _YouTubeModule extends CoreClass {
 							next => {
 								YouTubeModule.log(
 									"INFO",
-									`Getting channel progress for job (${this.toString()}): ${
-										songs.length
+									`Getting channel progress for job (${this.toString()}): ${songs.length
 									} songs gotten so far. Is there a next page: ${nextPageToken !== undefined}.`
 								);
 								next(null, nextPageToken !== undefined);
@@ -856,6 +864,50 @@ class _YouTubeModule extends CoreClass {
 			}
 		});
 	}
+
+	GET_API_REQUESTS(payload) {
+		return new Promise((resolve, reject) => {
+			const fromDate = payload.fromDate ? new Date(payload.fromDate) : new Date();
+
+			YouTubeModule.youtubeApiRequestModel
+				.find({ date: { $lte: fromDate } })
+				.sort({ date: -1 })
+				.exec((err, youtubeApiRequests) => {
+					if (err) reject(new Error("Couldn't load YouTube API requests."));
+					else {
+						resolve({ apiRequests: youtubeApiRequests });
+					}
+				});
+		});
+	}
+
+	GET_API_REQUEST(payload) {
+		return new Promise((resolve, reject) => {
+			const { apiRequestId } = payload;
+			// TODO validate apiRequestId
+			// TODO better error handling/waterfall
+
+			YouTubeModule.youtubeApiRequestModel
+				.findOne({ _id: apiRequestId })
+				.exec((err, apiRequest) => {
+					if (err) reject(new Error("Couldn't load YouTube API requests."));
+					else {
+						CacheModule.runJob("HGET", {
+							table: "youtubeApiRequestParams",
+							key: apiRequestId.toString()
+						}).then(apiRequestParams => {
+							resolve({
+								apiRequest: {
+									...apiRequest._doc,
+									params: apiRequestParams
+								}
+							});
+						});
+					}
+				});
+
+		});
+	}
 }
 
 export default new _YouTubeModule();

frontend/src/main.js

@@ -205,6 +205,10 @@ const router = createRouter({
 				{
 					path: "statistics",
 					component: () => import("@/pages/Admin/Statistics.vue")
+				},
+				{
+					path: "youtube",
+					component: () => import("@/pages/Admin/YouTube.vue")
 				}
 			],
 			meta: {

frontend/src/pages/Admin/YouTube.vue

@@ -0,0 +1,223 @@
+<template>
+	<div class="container">
+		<page-metadata title="Admin | YouTube" />
+		<div class="card">
+			<header class="card-header">
+				<p>Quota stats</p>
+			</header>
+			<div class="card-content">
+				<p v-if="fromDate">As of {{ fromDate }}</p>
+				<div
+					v-for="[quotaName, quotaObject] in Object.entries(
+						quotaStatus
+					)"
+					:key="quotaName"
+				>
+					<p>{{ quotaName }}</p>
+					<hr />
+					<p>Quota used: {{ quotaObject.quotaUsed }}</p>
+					<p>Limit: {{ quotaObject.limit }}</p>
+					<p>Quota exceeded: {{ quotaObject.quotaExceeded }}</p>
+					<br />
+				</div>
+			</div>
+		</div>
+		<br />
+		<div class="card">
+			<header class="card-header">
+				<p>API requests</p>
+			</header>
+			<div class="card-content">
+				<p v-if="fromDate">As of {{ fromDate }}</p>
+				<table class="table">
+					<thead>
+						<tr>
+							<th>Date</th>
+							<th>Quota cost</th>
+							<th>URL</th>
+							<th>Request ID</th>
+						</tr>
+					</thead>
+					<tbody>
+						<tr
+							v-for="apiRequest in apiRequests"
+							:key="apiRequest._id"
+						>
+							<td>
+								<router-link
+									:to="`?fromDate=${apiRequest.date}`"
+								>
+									{{ apiRequest.date }}
+								</router-link>
+							</td>
+							<td>{{ apiRequest.quotaCost }}</td>
+							<td>{{ apiRequest.url }}</td>
+							<td>
+								<router-link
+									:to="`?apiRequestId=${apiRequest._id}`"
+								>
+									{{ apiRequest._id }}
+								</router-link>
+							</td>
+						</tr>
+					</tbody>
+				</table>
+			</div>
+		</div>
+		<br />
+		<div class="card" v-if="currentApiRequest">
+			<header class="card-header">
+				<p>API request</p>
+			</header>
+			<div class="card-content">
+				<p><b>ID:</b> {{ currentApiRequest._id }}</p>
+				<p><b>URL:</b> {{ currentApiRequest.url }}</p>
+				<div>
+					<b>Params:</b>
+					<ul v-if="currentApiRequest.params">
+						<li
+							v-for="[paramKey, paramValue] in Object.entries(
+								currentApiRequest.params
+							)"
+							:key="paramKey"
+						>
+							<b>{{ paramKey }}</b
+							>: {{ paramValue }}
+						</li>
+					</ul>
+					<span v-else>None/Not found</span>
+				</div>
+				<p><b>Date:</b> {{ currentApiRequest.date }}</p>
+				<p><b>Quota cost:</b> {{ currentApiRequest.quotaCost }}</p>
+			</div>
+		</div>
+	</div>
+</template>
+
+<script>
+import { mapGetters } from "vuex";
+
+import ws from "@/ws";
+
+export default {
+	components: {},
+	data() {
+		return {
+			quotaStatus: {},
+			apiRequests: [],
+			currentApiRequest: null,
+			fromDate: null
+		};
+	},
+	computed: mapGetters({
+		socket: "websockets/getSocket"
+	}),
+	mounted() {
+		ws.onConnect(this.init);
+	},
+	methods: {
+		init() {
+			if (this.$route.query.fromDate)
+				this.fromDate = this.$route.query.fromDate;
+
+			this.socket.dispatch(
+				"youtube.getQuotaStatus",
+				this.fromDate,
+				res => {
+					if (res.status === "success")
+						this.quotaStatus = res.data.status;
+				}
+			);
+
+			this.socket.dispatch(
+				"youtube.getApiRequests",
+				this.fromDate,
+				res => {
+					if (res.status === "success")
+						this.apiRequests = res.data.apiRequests;
+				}
+			);
+
+			if (this.$route.query.apiRequestId) {
+				this.socket.dispatch(
+					"youtube.getApiRequest",
+					this.$route.query.apiRequestId,
+					res => {
+						if (res.status === "success")
+							this.currentApiRequest = res.data.apiRequest;
+					}
+				);
+			}
+		},
+		round(number) {
+			return Math.round(number);
+		}
+	}
+};
+</script>
+
+<style lang="less" scoped>
+.night-mode {
+	.table {
+		color: var(--light-grey-2);
+		background-color: var(--dark-grey-3);
+
+		thead tr {
+			background: var(--dark-grey-3);
+			td {
+				color: var(--white);
+			}
+		}
+
+		tbody tr:hover {
+			background-color: var(--dark-grey-4) !important;
+		}
+
+		tbody tr:nth-child(even) {
+			background-color: var(--dark-grey-2);
+		}
+
+		strong {
+			color: var(--light-grey-2);
+		}
+	}
+
+	.card {
+		background-color: var(--dark-grey-3);
+
+		p {
+			color: var(--light-grey-2);
+		}
+	}
+}
+
+td {
+	vertical-align: middle;
+}
+
+.is-primary:focus {
+	background-color: var(--primary-color) !important;
+}
+
+ul {
+	list-style-type: disc;
+	padding-left: 20px;
+}
+
+.card {
+	display: flex;
+	flex-grow: 1;
+	flex-direction: column;
+	padding: 20px;
+	margin: 10px;
+	border-radius: @border-radius;
+	background-color: var(--white);
+	color: var(--dark-grey);
+	box-shadow: @box-shadow;
+
+	.card-header {
+		font-weight: 700;
+		padding-bottom: 10px;
+	}
+}
+</style>

frontend/src/pages/Admin/index.vue

@@ -195,6 +195,18 @@
 								<i class="material-icons">show_chart</i>
 								<span>Statistics</span>
 							</router-link>
+							<router-link
+								class="sidebar-item youtube"
+								to="/admin/youtube"
+								content="YouTube"
+								v-tippy="{
+									theme: 'info',
+									onShow: () => !sidebarActive
+								}"
+							>
+								<i class="material-icons">circle</i>
+								<span>YouTube</span>
+							</router-link>
 						</div>
 					</div>
 				</div>