refactor: added back better support for type casting and worked on support for arrays with both normal and schema types

backend/src/collections/abc.ts

@@ -78,6 +78,11 @@ export const schema: AbcCollection = {
 					}
 				}
 			}
+		},
+		aNumber: {
+			type: Types.Number,
+			required: true,
+			restricted: false
 		}
 	},
 	timestamps: true,

backend/src/main.ts

@@ -54,22 +54,36 @@ global.rs = () => {
 // }, 3000);
 
 setTimeout(async () => {
+	const _id = "6371212daf4e9f8fb14444b2";
+
 	// logBook.log("Find with no projection");
 	// await moduleManager
 	// 	.runJob("data", "find", {
 	// 		collection: "abc",
 	// 		filter: {
-	// 			_id: "636fdc713450b25c3fc4ab0a"
+	// 			_id
+	// 		}
+	// 	})
+	// 	.then(console.log)
+	// 	.catch(console.error);
+
+	// logBook.log("Find with no projection, and a more advanced filter");
+	// await moduleManager
+	// 	.runJob("data", "find", {
+	// 		collection: "abc",
+	// 		filter: {
+	// 			"autofill.enabled": true
 	// 		}
 	// 	})
 	// 	.then(console.log)
 	// 	.catch(console.error);
+
 	// logBook.log("Find with array projection");
 	// await moduleManager
 	// 	.runJob("data", "find", {
 	// 		collection: "abc",
 	// 		filter: {
-	// 			_id: "636fdc713450b25c3fc4ab0a"
+	// 			_id
 	// 		},
 	// 		projection: ["name"]
 	// 	})
@@ -80,9 +94,10 @@ setTimeout(async () => {
 	// 	.runJob("data", "find", {
 	// 		collection: "abc",
 	// 		filter: {
-	// 			_id: "636fdc713450b25c3fc4ab0a"
+	// 			_id
 	// 		},
-	// 		projection: { name: true }
+	// 		projection: { name: true },
+	// 		limit: 1
 	// 	})
 	// 	.then(console.log)
 	// 	.catch(console.error);
@@ -91,7 +106,7 @@ setTimeout(async () => {
 	// 	.runJob("data", "find", {
 	// 		collection: "abc",
 	// 		filter: {
-	// 			_id: "636fdc713450b25c3fc4ab0a"
+	// 			_id
 	// 		},
 	// 		projection: { name: 1 }
 	// 	})
@@ -102,12 +117,28 @@ setTimeout(async () => {
 	// 	.runJob("data", "find", {
 	// 		collection: "abc",
 	// 		filter: {
-	// 			_id: "636fdc713450b25c3fc4ab0a"
+	// 			_id
 	// 		},
 	// 		projection: { "autofill.enabled": true }
 	// 	})
 	// 	.then(console.log)
 	// 	.catch(console.error);
+
+	// logBook.log("Find for testing casting");
+	// await moduleManager
+	// 	.runJob("data", "find", {
+	// 		collection: "abc",
+	// 		filter: {
+	// 			_id
+	// 		},
+	// 		// projection: {
+	// 		// 	// songs: true,
+	// 		// 	// someNumbers: false
+	// 		// },
+	// 		limit: 1
+	// 	})
+	// 	.then(console.log)
+	// 	.catch(console.error);
 }, 0);
 
 const rl = readline.createInterface({

backend/src/modules/DataModule.spec.ts

@@ -36,6 +36,12 @@ describe("Data Module", function () {
 				autofill: {
 					enabled: !!Math.floor(Math.random())
 				},
+				someNumbers: Array(Math.round(Math.random() * 50)).map(() =>
+					Math.round(Math.random() * 10000)
+				),
+				songs: Array(Math.round(Math.random() * 10)).map(() => ({
+					_id: new mongoose.Types.ObjectId()
+				})),
 				createdAt: Date.now(),
 				updatedAt: Date.now(),
 				testData: true
@@ -64,7 +70,7 @@ describe("Data Module", function () {
 					useCache
 				});
 
-				find.should.be.a("object");
+				find.should.be.an("object");
 				find._id.should.deep.equal(document._id);
 				find.createdAt.should.deep.equal(document.createdAt);
 
@@ -73,6 +79,29 @@ describe("Data Module", function () {
 				}
 			});
 		});
+
+		it(`filter by name string without cache`, async function () {
+			const [document] = testData.abc;
+
+			const find = await dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { name: document.name },
+				limit: 1,
+				useCache: false
+			});
+
+			find.should.be.an("object");
+			find._id.should.deep.equal(document._id);
+			find.should.have.keys([
+				"_id",
+				"createdAt",
+				"updatedAt",
+				// "name", - Name is restricted, so it won't be returned
+				"autofill",
+				"someNumbers",
+				"songs"
+			]);
+		});
 	});
 
 	afterEach(async function () {

backend/src/modules/DataModule.ts

@@ -1,14 +1,14 @@
 // @ts-nocheck
 import async from "async";
 import config from "config";
-import { Db, MongoClient } from "mongodb";
+import { Db, MongoClient, ObjectId } from "mongodb";
 import hash from "object-hash";
 import { createClient, RedisClientType } from "redis";
 import JobContext from "src/JobContext";
 import BaseModule from "../BaseModule";
 import ModuleManager from "../ModuleManager";
 import { UniqueMethods } from "../types/Modules";
-import { Collections } from "../types/Collections";
+import { Collections, Types } from "../types/Collections";
 
 export default class DataModule extends BaseModule {
 	private collections?: Collections;
@@ -253,10 +253,29 @@ export default class DataModule extends BaseModule {
 	 * @returns
 	 */
 	private allowedByProjection(projection: any, property: string) {
+		let topLevelKeys = [];
+
 		if (Array.isArray(projection))
-			return projection.indexOf(property) !== -1;
-		if (typeof projection === "object") return !!projection[property];
-		return false;
+			topLevelKeys = projection.map(key => [key, true]);
+		else if (typeof projection === "object")
+			topLevelKeys = Object.entries(projection);
+
+		// Turn a list of properties like ["propertyName", "propertyNameTwo.nestedProperty", "propertyName.test"] and into ["propertyName", "propertyNameTwo"]
+		topLevelKeys = topLevelKeys.reduce((arr, [key, value]) => {
+			let normalizedKey = key;
+			if (normalizedKey.indexOf(".") !== -1)
+				normalizedKey = normalizedKey.substr(
+					0,
+					normalizedKey.indexOf(".")
+				);
+			if (arr.indexOf(normalizedKey) === -1)
+				return [...arr, [normalizedKey, value]];
+			return arr;
+		}, []);
+
+		topLevelKeys = Object.fromEntries(topLevelKeys);
+
+		return !!topLevelKeys[property];
 	}
 
 	/**
@@ -271,6 +290,8 @@ export default class DataModule extends BaseModule {
 	private async stripDocument(document: any, schema: any, projection: any) {
 		// TODO add better comments
 		// TODO add support for nested objects in arrays
+		// TODO possibly do different things with required properties?
+		// TODO possibly do different things with properties with default?
 		// TODO handle projection excluding properties, rather than assume it's only including properties
 
 		const unfilteredEntries = Object.entries(document);
@@ -278,13 +299,24 @@ export default class DataModule extends BaseModule {
 			unfilteredEntries,
 			[],
 			async (memo, [key, value]) => {
-				// If the property does not exist in the schema, return the memo
+				// If the property does not exist in the schema, return the memo, so we won't return the key/value in the stripped document
 				if (!schema[key]) return memo;
 
+				// If we have a projection, check if the current key is allowed by it. If it not, just return the memo
+				if (projection) {
+					const allowedByProjection = this.allowedByProjection(
+						projection,
+						key
+					);
+
+					if (!allowedByProjection) return memo;
+				}
+
 				// Handle nested object
-				if (schema[key].type === undefined) {
-					// If value is null, it can't be an object, so just return its value
-					if (!value) return [...memo, [key, value]];
+				if (schema[key].type === Types.Schema) {
+					// TODO possibly return nothing, or an empty object here instead?
+					// If value is falsy, it can't be an object, so just return null
+					if (!value) return [...memo, [key, null]];
 
 					// Get the projection for the next layer
 					const deeperProjection = this.getDeeperProjection(
@@ -295,7 +327,7 @@ export default class DataModule extends BaseModule {
 					// Generate a stripped document/object for the current key/value
 					const strippedDocument = await this.stripDocument(
 						value,
-						schema[key],
+						schema[key].schema,
 						deeperProjection
 					);
 
@@ -303,21 +335,90 @@ export default class DataModule extends BaseModule {
 					if (Object.keys(strippedDocument).length > 0)
 						return [...memo, [key, strippedDocument]];
 
+					// TODO possibly return null or an object here for the key instead?
 					// The current key has no values that should be returned, so just return the memo
 					return memo;
 				}
 
-				// If we have a projection, check if the current key is allowed by it. If it is, add the key/value to the memo, otherwise just return the memo
-				if (projection)
-					return this.allowedByProjection(projection, key)
-						? [...memo, [key, value]]
-						: memo;
+				// Handle array type
+				if (schema[key].type === Types.Array) {
+					// TODO possibly return nothing, or an empty array here instead?
+					// If value is falsy, return null with the key instead
+					if (!value) return [...memo, [key, null]];
+
+					// TODO possibly return nothing, or an empty array here instead?
+					// If value isn't a valid array, return null with the key instead
+					if (!Array.isArray(value)) return [...memo, [key, null]];
+
+					// The type of the array items
+					const itemType = schema[key].item.type;
+
+					const items = await async.map(value, async item => {
+						// Handle schema objects inside an array
+						if (itemType === Types.Schema) {
+							// TODO possibly return nothing, or an empty object here instead?
+							// If item is falsy, it can't be an object, so just return null
+							if (!item) return null;
+
+							// Get the projection for the next layer
+							const deeperProjection = this.getDeeperProjection(
+								projection,
+								key
+							);
+
+							// Generate a stripped document/object for the current key/value
+							const strippedDocument = await this.stripDocument(
+								item,
+								schema[key].item.schema,
+								deeperProjection
+							);
+
+							// If the returned stripped document/object has keys, return the stripped document
+							if (Object.keys(strippedDocument).length > 0)
+								return strippedDocument;
+
+							// TODO possibly return object here instead?
+							// The current item has no values that should be returned, so just return null
+							return null;
+						}
+						// Nested arrays are not supported
+						if (itemType === Types.Array) {
+							throw new Error("Nested arrays not supported");
+						}
+						// Handle normal types
+						else {
+							// If item is null or undefined, return null
+							const isNullOrUndefined =
+								item === null || item === undefined;
+							if (isNullOrUndefined) return null;
+
+							// TODO possibly don't validate casted?
+							// Cast item
+							const castedValue = this.getCastedValue(
+								item,
+								itemType
+							);
+
+							return castedValue;
+						}
+					});
+
+					return [...memo, [key, items]];
+				}
 
 				// If the property is restricted, return memo
 				if (schema[key].restricted) return memo;
 
 				// The property exists in the schema, is not explicitly allowed, is not restricted, so add it to memo
-				return [...memo, [key, value]];
+				// Add type casting here
+
+				// TODO possible don't validate casted?
+				const castedValue = this.getCastedValue(
+					value,
+					schema[key].type
+				);
+
+				return [...memo, [key, castedValue]];
 			}
 		);
 
@@ -358,8 +459,8 @@ export default class DataModule extends BaseModule {
 				else if (restricted) {
 					mongoProjection[key] = false;
 				}
-				// If the current property is a nested object
-				else if (value.type === undefined) {
+				// If the current property is a nested schema
+				else if (value.type === Types.Schema) {
 					// Get the projection for the next layer
 					const deeperProjection = this.getDeeperProjection(
 						projection,
@@ -385,7 +486,7 @@ export default class DataModule extends BaseModule {
 			// If we have no projection set, and the current property is restricted, exclude the property from mongo, but don't say we can't use the cache
 			else if (value.restricted) mongoProjection[key] = false;
 			// If we have no projection set, and the current property is not restricted, and the current property is a nested object
-			else if (value.type === undefined) {
+			else if (value.type === Types.Schema) {
 				// Pass the nested schema object recursively into the parseFindProjection function
 				const parsedProjection = await this.parseFindProjection(
 					null,
@@ -406,6 +507,58 @@ export default class DataModule extends BaseModule {
 		};
 	}
 
+	private getCastedValue(value, schemaType) {
+		if (schemaType === Types.String) {
+			// Check if value is a string, and if not, convert the value to a string
+			const castedValue =
+				typeof value === "string" ? value : String(value);
+			// Any additional validation comes here
+			return castedValue;
+		}
+		if (schemaType === Types.Number) {
+			// Check if value is a number, and if not, convert the value to a number
+			const castedValue =
+				typeof value === "number" ? value : Number(value);
+			// TODO possibly allow this via a validate boolean option?
+			// We don't allow NaN for numbers, so throw an error
+			if (Number.isNaN(castedValue))
+				throw new Error(
+					`Cast error, number cannot be NaN, at key ${key} with value ${value}`
+				);
+			// Any additional validation comes here
+			return castedValue;
+		}
+		if (schemaType === Types.Date) {
+			// Check if value is a Date, and if not, convert the value to a Date
+			const castedValue =
+				Object.prototype.toString.call(value) === "[object Date]"
+					? value
+					: new Date(value);
+			// TODO possibly allow this via a validate boolean option?
+			// We don't allow invalid dates, so throw an error
+			// eslint-disable-next-line
+			if (isNaN(castedValue)) throw new Error(`Cast error, date cannot be invalid, at key ${key} with value ${value}`);
+			// Any additional validation comes here
+			return castedValue;
+		}
+		if (schemaType === Types.Boolean) {
+			// Check if value is a boolean, and if not, convert the value to a boolean
+			const castedValue =
+				typeof value === "boolean" ? value : Boolean(value);
+			// Any additional validation comes here
+			return castedValue;
+		}
+		if (schemaType === Types.ObjectId) {
+			// Cast the value as an ObjectId and let Mongoose handle the rest
+			const castedValue = ObjectId(value);
+			// Any additional validation comes here
+			return castedValue;
+		}
+		throw new Error(
+			`Unsupported schema type found with type ${Types[schemaType]}. This should never happen.`
+		);
+	}
+
 	/**
 	 * parseFindFilter - Ensure validity of filter and return a mongo filter ---, or the document itself re-constructed
 	 *
@@ -472,13 +625,13 @@ export default class DataModule extends BaseModule {
 					// $or and $and should always be an array, so check if it is
 					if (!Array.isArray(value) || value.length === 0)
 						throw new Error(
-							`Key "${key}" must contain array of queries.`
+							`Key "${key}" must contain array of filters.`
 						);
 
 					// Add the operator to the mongo filter object as an empty array
 					mongoFilter[key] = [];
 
-					// Run parseFindQuery again for child objects and add them to the mongo query operator array
+					// Run parseFindQuery again for child objects and add them to the mongo filter operator array
 					await async.each(value, async _value => {
 						const {
 							mongoFilter: _mongoFilter,
@@ -486,7 +639,7 @@ export default class DataModule extends BaseModule {
 								_containsRestrictedProperties
 						} = await this.parseFindFilter(_value, schema, options);
 
-						// Actually add the returned filter object to the mongo query we're building
+						// Actually add the returned filter object to the mongo filter we're building
 						mongoFilter[key].push(_mongoFilter);
 						if (_containsRestrictedProperties)
 							containsRestrictedProperties = true;
@@ -507,82 +660,119 @@ export default class DataModule extends BaseModule {
 				// If the key in the schema is marked as restricted, containsRestrictedProperties will be true
 				if (schema[key].restricted) containsRestrictedProperties = true;
 
-				// Type will be undefined if it's a nested object
-				if (schema[key].type === undefined) {
+				// Handle schema type
+				if (schema[key].type === Types.Schema) {
 					// Run parseFindFilter on the nested schema object
 					const {
 						mongoFilter: _mongoFilter,
 						containsRestrictedProperties:
 							_containsRestrictedProperties
-					} = await this.parseFindFilter(value, schema[key], options);
+					} = await this.parseFindFilter(
+						value,
+						schema[key].schema,
+						options
+					);
 					mongoFilter[key] = _mongoFilter;
 					if (_containsRestrictedProperties)
 						containsRestrictedProperties = true;
-				} else if (
-					operators &&
-					typeof value === "object" &&
-					value &&
-					Object.keys(value).length === 1 &&
-					Object.keys(value)[0] &&
-					Object.keys(value)[0][0] === "$"
-				) {
-					// This entire if statement is for handling value operators
-
-					const operator = Object.keys(value)[0];
-
-					// Operator isn't found, so throw an error
-					if (allowedValueOperators.indexOf(operator) === -1)
+				}
+				// Handle array type
+				else if (schema[key].type === Types.Array) {
+					// // Run parseFindFilter on the nested schema object
+					// const {
+					// 	mongoFilter: _mongoFilter,
+					// 	containsRestrictedProperties:
+					// 		_containsRestrictedProperties
+					// } = await this.parseFindFilter(
+					// 	value,
+					// 	schema[key].schema,
+					// 	options
+					// );
+					// mongoFilter[key] = _mongoFilter;
+					// if (_containsRestrictedProperties)
+					// 	containsRestrictedProperties = true;
+
+					throw new Error("NOT SUPPORTED YET.");
+				}
+				// else if (
+				// 	operators &&
+				// 	typeof value === "object" &&
+				// 	value &&
+				// 	Object.keys(value).length === 1 &&
+				// 	Object.keys(value)[0] &&
+				// 	Object.keys(value)[0][0] === "$"
+				// ) {
+				// 	// This entire if statement is for handling value operators
+
+				// 	const operator = Object.keys(value)[0];
+
+				// 	// Operator isn't found, so throw an error
+				// 	if (allowedValueOperators.indexOf(operator) === -1)
+				// 		throw new Error(
+				// 			`Invalid filter provided. Operator "${key}" is not allowed.`
+				// 		);
+
+				// 	// Handle the $in value operator
+				// 	if (operator === "$in") {
+				// 		mongoFilter[key] = {
+				// 			$in: []
+				// 		};
+
+				// 		if (value.$in.length > 0)
+				// 			mongoFilter[key].$in = await async.map(
+				// 				value.$in,
+				// 				async (_value: any) => {
+				// 					// if (
+				// 					// 	typeof schema[key].type === "function"
+				// 					// ) {
+				// 					// 	//
+				// 					// 	// const Type = schema[key].type;
+				// 					// 	// const castValue = new Type(_value);
+				// 					// 	// if (schema[key].validate)
+				// 					// 	// 	await schema[key]
+				// 					// 	// 		.validate(castValue)
+				// 					// 	// 		.catch(err => {
+				// 					// 	// 			throw new Error(
+				// 					// 	// 				`Invalid value for ${key}, ${err}`
+				// 					// 	// 			);
+				// 					// 	// 		});
+				// 					// 	return _value;
+				// 					// }
+				// 					// throw new Error(
+				// 					// 	`Invalid schema type for ${key}`
+				// 					// );
+				// 					console.log(_value);
+
+				// 					return _value;
+				// 				}
+				// 			);
+				// 	}
+				// else
+				// 	throw new Error(
+				// 		`Unhandled operator "${operator}", this should never happen!`
+				// 	);
+				// }
+				// Handle normal types
+				else {
+					const isNullOrUndefined =
+						value === null || value === undefined;
+					if (isNullOrUndefined && schema[key].required)
 						throw new Error(
-							`Invalid filter provided. Operator "${key}" is not allowed.`
+							`Value for key ${key} is required, so it cannot be null/undefined.`
 						);
 
-					// Handle the $in value operator
-					if (operator === "$in") {
-						mongoFilter[key] = {
-							$in: []
-						};
-
-						if (value.$in.length > 0)
-							mongoFilter[key].$in = await async.map(
-								value.$in,
-								async (_value: any) => {
-									if (
-										typeof schema[key].type === "function"
-									) {
-										//
-										// const Type = schema[key].type;
-										// const castValue = new Type(_value);
-										// if (schema[key].validate)
-										// 	await schema[key]
-										// 		.validate(castValue)
-										// 		.catch(err => {
-										// 			throw new Error(
-										// 				`Invalid value for ${key}, ${err}`
-										// 			);
-										// 		});
-										return _value;
-									}
-									throw new Error(
-										`Invalid schema type for ${key}`
-									);
-								}
-							);
-					} else
-						throw new Error(
-							`Unhandled operator "${operator}", this should never happen!`
+					// If the value is null or undefined, just set it as null
+					if (isNullOrUndefined) mongoFilter[key] = null;
+					// Cast and validate values
+					else {
+						const schemaType = schema[key].type;
+
+						mongoFilter[key] = this.getCastedValue(
+							value,
+							schemaType
 						);
-				} else if (typeof schema[key].type === "function") {
-					// Do type checking/casting here
-
-					// const Type = schema[key].type;
-					// // const castValue = new Type(value);
-					// if (schema[key].validate)
-					// 	await schema[key].validate(castValue).catch(err => {
-					// 		throw new Error(`Invalid value for ${key}, ${err}`);
-					// 	});
-
-					mongoFilter[key] = value;
-				} else throw new Error(`Invalid schema type for ${key}`);
+					}
+				}
 			}
 		});