refactor: changed the DataModule parseQuery function and find job function a little bit, and a few other small changes

.vscode/settings.json

@@ -1,3 +1,2 @@
 {
-	"editor.defaultFormatter": "rvest.vs-code-prettier-eslint"
 }

backend/src/ModuleManager.ts

@@ -196,7 +196,7 @@ export default class ModuleManager {
 								this.logBook,
 								job
 							);
-							jobFunction(jobContext, payload)
+							jobFunction.apply(module, [jobContext, payload])
 								.then((response: ReturnType) => {
 									this.logBook.log({
 										message: "Job completed successfully",

backend/src/main.ts

@@ -16,21 +16,23 @@ global.rs = () => {
 	process.exit();
 };
 
-const interval = setInterval(() => {
-	moduleManager
-		.runJob("stations", "addToQueue", { songId: "TestId" })
-		.catch(() => {});
-	moduleManager
-		.runJob("stations", "addA", void 0, { priority: 5 })
-		.catch(() => {});
-	moduleManager
-		.runJob("others", "doThing", { test: "Test", test2: 123 })
-		.catch(() => {});
-}, 40);
-
-setTimeout(() => {
-	clearTimeout(interval);
-}, 3000);
+// const interval = setInterval(() => {
+// 	moduleManager
+// 		.runJob("stations", "addToQueue", { songId: "TestId" })
+// 		.catch(() => {});
+// 	moduleManager
+// 		.runJob("stations", "addA", void 0, { priority: 5 })
+// 		.catch(() => {});
+// 	moduleManager
+// 		.runJob("others", "doThing", { test: "Test", test2: 123 })
+// 		.catch(() => {});
+// }, 40);
+
+// setTimeout(() => {
+// 	clearTimeout(interval);
+// }, 3000);
+
+
 
 // Temp fix
 process.removeAllListeners("uncaughtException");

backend/src/modules/DataModule.ts

@@ -3,11 +3,11 @@ import config from "config";
 import mongoose, { Schema } from "mongoose";
 import hash from "object-hash";
 import { createClient, RedisClientType } from "redis";
+import JobContext from "src/JobContext";
 import BaseModule from "../BaseModule";
 import ModuleManager from "../ModuleManager";
 import { UniqueMethods } from "../types/Modules";
 import { Collections } from "../types/Collections";
-import JobContext from "src/JobContext";
 
 export default class DataModule extends BaseModule {
 	collections?: Collections;
@@ -174,7 +174,7 @@ export default class DataModule extends BaseModule {
 
 	// TODO split core into parseDocument(document, schema, { partial: boolean;  })
 	/**
-	 * parseQuery - Ensure validity of query
+	 * parseQuery - Ensure validity of query and return a mongo query, or the document itself re-constructed
 	 *
 	 * @param query - Query
 	 * @param schema - Schema of collection document
@@ -191,44 +191,107 @@ export default class DataModule extends BaseModule {
 	): Promise<{ castQuery: any; restricted: boolean }> {
 		if (!query || typeof query !== "object")
 			throw new Error("Invalid query provided. Query must be an object.");
+
 		const keys = Object.keys(query);
 		if (keys.length === 0)
 			throw new Error("Invalid query provided. Query must contain keys.");
+
+		// Whether to parse operators or not
 		const operators = !(options && options.operators === false);
+		// The MongoDB query we're building
 		const castQuery: any = {};
+		// If the query references any fields that are restricted, this will be true, so that find knows not to cache the query object
 		let restricted = false;
+
+		// Operators at the key level that we support right now
+		const allowedKeyOperators = ["$or", "$and"];
+		// Operators at the value level that we support right now
+		const allowedValueOperators = ["$in"];
+
 		await async.each(Object.entries(query), async ([key, value]) => {
-			if (operators && (key === "$or" || key === "$and")) {
-				if (!Array.isArray(value))
+			// Key must be 1 character and exist
+			if (!key || key.length === 0)
+				throw new Error(
+					`Invalid query provided. Key must be at least 1 character.`
+				);
+
+			// Handle key operators, which always start with a $
+			if (operators && key[0] === "$") {
+				// Operator isn't found, so throw an error
+				if (allowedKeyOperators.indexOf(key) === -1)
 					throw new Error(
-						`Key "${key}" must contain array of queries`
+						`Invalid query provided. Operator "${key}" is not allowed.`
 					);
-				castQuery[key] = [];
-				await async.each(value, async _value => {
-					const { castQuery: _castQuery, restricted: _restricted } =
-						await this.parseQuery(_value, schema, options);
-					castQuery[key].push(_castQuery);
-					restricted = restricted || _restricted;
-				});
-			} else if (schema[key] !== undefined) {
+
+				// We currently only support $or and $and, but here we can have different logic for different operators
+				if (key === "$or" || key === "$and") {
+					// $or and $and should always be an array, so check if it is
+					if (!Array.isArray(value) || value.length === 0)
+						throw new Error(
+							`Key "${key}" must contain array of queries.`
+						);
+
+					// Add the operator to the mongo query object as an empty array
+					castQuery[key] = [];
+
+					// Run parseQuery again for child objects and add them to the mongo query operator array
+					await async.each(value, async _value => {
+						const {
+							castQuery: _castQuery,
+							restricted: _restricted
+						} = await this.parseQuery(_value, schema, options);
+
+						// Actually add the returned query object to the mongo query we're building
+						castQuery[key].push(_castQuery);
+						if (_restricted) restricted = true;
+					});
+				} else
+					throw new Error(
+						`Unhandled operator "${key}", this should never happen!`
+					);
+			} else {
+				// Here we handle any normal keys in the query object
+
+				// If the key doesn't exist in the schema, throw an error
+				if (!Object.hasOwn(schema, key))
+					throw new Error(
+						`Key "${key} does not exist in the schema."`
+					);
+
+				// If the key in the schema is marked as restricted, mark the entire query as restricted
 				if (schema[key].restricted) restricted = true;
-				if (
-					schema[key].type === undefined &&
-					Object.keys(schema[key]).length > 0
-				) {
+
+				// Type will be undefined if it's a nested object
+				if (schema[key].type === undefined) {
+					// Run parseQuery on the nested schema object
 					const { castQuery: _castQuery, restricted: _restricted } =
 						await this.parseQuery(value, schema[key], options);
 					castQuery[key] = _castQuery;
-					restricted = restricted || _restricted;
+					if (_restricted) restricted = true;
 				} else if (
 					operators &&
 					typeof value === "object" &&
+					value &&
 					Object.keys(value).length === 1 &&
-					Array.isArray(value.$in)
+					Object.keys(value)[0] &&
+					Object.keys(value)[0][0] === "$"
 				) {
-					if (value.$in.length > 0)
+					// This entire if statement is for handling value operators
+
+					// Operator isn't found, so throw an error
+					if (allowedValueOperators.indexOf(key) === -1)
+						throw new Error(
+							`Invalid query provided. Operator "${key}" is not allowed.`
+						);
+
+					// Handle the $in value operator
+					if (value.$in) {
 						castQuery[key] = {
-							$in: await async.map(
+							$in: []
+						};
+
+						if (value.$in.length > 0)
+							castQuery[key].$in = await async.map(
 								value.$in,
 								async (_value: any) => {
 									if (
@@ -250,9 +313,13 @@ export default class DataModule extends BaseModule {
 										`Invalid schema type for ${key}`
 									);
 								}
-							)
-						};
-					else throw new Error(`Invalid value for ${key}`);
+							);
+					} else
+						throw new Error(
+							`Unhandled operator "${
+								Object.keys(value)[0]
+							}", this should never happen!`
+						);
 				} else if (typeof schema[key].type === "function") {
 					const Type = schema[key].type;
 					const castValue = new Type(value);
@@ -262,12 +329,9 @@ export default class DataModule extends BaseModule {
 						});
 					castQuery[key] = castValue;
 				} else throw new Error(`Invalid schema type for ${key}`);
-			} else {
-				throw new Error(
-					`Invalid query provided. Key "${key}" not found`
-				);
 			}
 		});
+
 		return { castQuery, restricted };
 	}
 
@@ -284,7 +348,7 @@ export default class DataModule extends BaseModule {
 	// TODO prevent caching if requiring restricted values
 	// TODO fix 2nd layer of schema
 	/**
-	 * find - Find data
+	 * find - Get one or more document(s) from a single collection
 	 *
 	 * @param payload - Payload
 	 * @returns Returned object
@@ -292,8 +356,8 @@ export default class DataModule extends BaseModule {
 	public find<T extends keyof Collections>(
 		context: JobContext,
 		{
-			collection,
-			query,
+			collection, // Collection name
+			query, // Similar to MongoDB query
 			values, // TODO: Add support
 			limit = 1, // TODO have limit off by default?
 			page = 1,
@@ -328,26 +392,32 @@ export default class DataModule extends BaseModule {
 							this.collections![collection].schema.document
 						),
 
-					// If we can use cache, get from the cache, and if we get results return those, otherwise return null
+					// If we can use cache, get from the cache, and if we get results return those
 					async ({ castQuery, restricted }: any) => {
-						// Not using cache or query contains restricted values, so return
-						if (!cacheable || restricted)
-							return { castQuery, cachedDocuments: null };
-						queryHash = hash(
-							{ collection, castQuery, values, limit, page },
-							{
-								algorithm: "sha1"
-							}
-						);
-						const cachedQuery = await this.redis?.GET(
-							`query.find.${queryHash}`
-						);
-						return {
-							castQuery,
-							cachedDocuments: cachedQuery
-								? JSON.parse(cachedQuery)
-								: null
-						};
+						// If we're allowed to cache, and the query doesn't reference any restricted fields, try to cache the query and its response
+						if (cacheable && !restricted) {
+							// Turn the query object into a sha1 hash that can be used as a Redis key
+							queryHash = hash(
+								{ collection, castQuery, values, limit, page },
+								{
+									algorithm: "sha1"
+								}
+							);
+							// Check if the query hash already exists in Redis, and get it if it is
+							const cachedQuery = await this.redis?.GET(
+								`query.find.${queryHash}`
+							);
+
+							// Return the castQuery along with the cachedDocuments, if any
+							return {
+								castQuery,
+								cachedDocuments: cachedQuery
+									? JSON.parse(cachedQuery)
+									: null
+							};
+						}
+
+						return { castQuery, cachedDocuments: null };
 					},
 
 					// If we didn't get documents from the cache, get them from mongo