refactor: added more support for DataModule find job filtering with arrays/schemas

backend/package-lock.json

@@ -42,6 +42,7 @@
 				"@typescript-eslint/eslint-plugin": "^5.40.0",
 				"@typescript-eslint/parser": "^5.40.0",
 				"chai": "^4.3.7",
+				"chai-as-promised": "^7.1.1",
 				"eslint": "^8.25.0",
 				"eslint-config-airbnb-base": "^15.0.0",
 				"eslint-config-prettier": "^8.5.0",
@@ -1089,6 +1090,18 @@
 				"node": ">=4"
 			}
 		},
+		"node_modules/chai-as-promised": {
+			"version": "7.1.1",
+			"resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
+			"integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
+			"dev": true,
+			"dependencies": {
+				"check-error": "^1.0.2"
+			},
+			"peerDependencies": {
+				"chai": ">= 2.1.2 < 5"
+			}
+		},
 		"node_modules/chalk": {
 			"version": "4.1.2",
 			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -5793,6 +5806,15 @@
 				"type-detect": "^4.0.5"
 			}
 		},
+		"chai-as-promised": {
+			"version": "7.1.1",
+			"resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
+			"integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
+			"dev": true,
+			"requires": {
+				"check-error": "^1.0.2"
+			}
+		},
 		"chalk": {
 			"version": "4.1.2",
 			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",

backend/package.json

@@ -50,6 +50,7 @@
 		"@typescript-eslint/eslint-plugin": "^5.40.0",
 		"@typescript-eslint/parser": "^5.40.0",
 		"chai": "^4.3.7",
+		"chai-as-promised": "^7.1.1",
 		"eslint": "^8.25.0",
 		"eslint-config-airbnb-base": "^15.0.0",
 		"eslint-config-prettier": "^8.5.0",

backend/src/main.ts

@@ -124,21 +124,29 @@ setTimeout(async () => {
 	// 	.then(console.log)
 	// 	.catch(console.error);
 
-	logBook.log("Find for testing casting");
-	await moduleManager
-		.runJob("data", "find", {
-			collection: "abc",
-			filter: {
-				_id
-			},
-			// projection: {
-			// 	// songs: true,
-			// 	// someNumbers: false
-			// },
-			limit: 1
-		})
-		.then(console.log)
-		.catch(console.error);
+	// logBook.log("Find for testing casting");
+	// await moduleManager
+	// 	.runJob("data", "find", {
+	// 		collection: "abc",
+	// 		filter: {
+	// 			// "songs._id": "6371212daf4e9f8fb14444b0"
+	// 			// "songs._id": "6371212daf4e9f8fb14444b2"
+	// 			// songs: {
+	// 			// 	_id: "6371212daf4e9f8fb14444b0"
+	// 			// }
+	// 			// songs: {
+	// 			// 	randomProperty: "6371212daf4e9f8fb14444b0"
+	// 			// }
+	// 			"songs.obj.test": "6371212daf4e9f8fb14444b0"
+	// 		},
+	// 		// projection: {
+	// 		// 	// songs: true,
+	// 		// 	// someNumbers: false
+	// 		// },
+	// 		limit: 1
+	// 	})
+	// 	.then(console.log)
+	// 	.catch(console.error);
 }, 0);
 
 const rl = readline.createInterface({

backend/src/modules/DataModule.spec.ts

@@ -3,6 +3,7 @@ import async from "async";
 import chai, { expect } from "chai";
 import sinon from "sinon";
 import sinonChai from "sinon-chai";
+import chaiAsPromised from "chai-as-promised";
 import { ObjectId } from "mongodb";
 import JobContext from "../JobContext";
 import JobQueue from "../JobQueue";
@@ -12,6 +13,7 @@ import DataModule from "./DataModule";
 
 chai.should();
 chai.use(sinonChai);
+chai.use(chaiAsPromised);
 
 describe("Data Module", function () {
 	const moduleManager = Object.getPrototypeOf(
@@ -29,24 +31,30 @@ describe("Data Module", function () {
 	});
 
 	beforeEach(async function () {
-		testData.abc = await async.map(Array(10), async () =>
-			dataModule.collections?.abc.collection.insertOne({
-				_id: new ObjectId(),
-				name: `Test${Math.round(Math.random() * 1000)}`,
-				autofill: {
-					enabled: !!Math.floor(Math.random())
-				},
-				someNumbers: Array(Math.round(Math.random() * 50)).map(() =>
-					Math.round(Math.random() * 10000)
-				),
-				songs: Array(Math.round(Math.random() * 10)).map(() => ({
-					_id: new mongoose.Types.ObjectId()
-				})),
-				createdAt: Date.now(),
-				updatedAt: Date.now(),
-				testData: true
-			})
-		);
+		testData.abc = await async.map(Array(10), async () => {
+			const result =
+				await dataModule.collections?.abc.collection.insertOne({
+					_id: new ObjectId(),
+					name: `Test${Math.round(Math.random() * 1000)}`,
+					autofill: {
+						enabled: !!Math.floor(Math.random())
+					},
+					someNumbers: Array.from({
+						length: Math.max(1, Math.round(Math.random() * 50))
+					}).map(() => Math.round(Math.random() * 10000)),
+					songs: Array.from({
+						length: Math.max(1, Math.round(Math.random() * 10))
+					}).map(() => ({
+						_id: new ObjectId()
+					})),
+					createdAt: Date.now(),
+					updatedAt: Date.now(),
+					testData: true
+				});
+			return dataModule.collections?.abc.collection.findOne({
+				_id: result?.insertedId
+			});
+		});
 	});
 
 	it("module loaded and started", function () {
@@ -72,7 +80,7 @@ describe("Data Module", function () {
 
 				find.should.be.an("object");
 				find._id.should.deep.equal(document._id);
-				find.createdAt.should.deep.equal(document.createdAt);
+				find.createdAt.should.deep.equal(new Date(document.createdAt));
 
 				if (useCache) {
 					dataModule.redisClient?.GET.should.have.been.called;
@@ -102,6 +110,85 @@ describe("Data Module", function () {
 				"songs"
 			]);
 		});
+
+		it(`filter by normal array item`, async function () {
+			const [document] = testData.abc;
+
+			const resultDocument = await dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { someNumbers: document.someNumbers[0] },
+				limit: 1,
+				useCache: false
+			});
+
+			resultDocument.should.be.an("object");
+			resultDocument._id.should.deep.equal(document._id);
+		});
+
+		it(`filter by normal array item that doesn't exist`, async function () {
+			const resultDocument = await dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { someNumbers: -1 },
+				limit: 1,
+				useCache: false
+			});
+
+			expect(resultDocument).to.be.null;
+		});
+
+		it(`filter by schema array item`, async function () {
+			const [document] = testData.abc;
+
+			const resultDocument = await dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { songs: { _id: document.songs[0]._id } },
+				limit: 1,
+				useCache: false
+			});
+
+			resultDocument.should.be.an("object");
+			resultDocument._id.should.deep.equal(document._id);
+		});
+
+		it(`filter by schema array item, invalid`, async function () {
+			const jobPromise = dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { songs: { randomProperty: "Value" } },
+				limit: 1,
+				useCache: false
+			});
+
+			await expect(jobPromise).to.be.rejectedWith(
+				`Key "randomProperty" does not exist in the schema.`
+			);
+		});
+
+		it(`filter by schema array item with dot notation`, async function () {
+			const [document] = testData.abc;
+
+			const resultDocument = await dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { "songs._id": document.songs[0]._id },
+				limit: 1,
+				useCache: false
+			});
+
+			resultDocument.should.be.an("object");
+			resultDocument._id.should.deep.equal(document._id);
+		});
+
+		it(`filter by schema array item with dot notation, invalid`, async function () {
+			const jobPromise = dataModule.find(jobContext, {
+				collection: "abc",
+				filter: { "songs.randomProperty": "Value" },
+				limit: 1,
+				useCache: false
+			});
+
+			await expect(jobPromise).to.be.rejectedWith(
+				`Key "randomProperty" does not exist in the schema.`
+			);
+		});
 	});
 
 	describe("normalize projection", function () {

backend/src/modules/DataModule.ts

@@ -542,6 +542,8 @@ export default class DataModule extends BaseModule {
 		containsRestrictedProperties: boolean;
 		canCache: boolean;
 	}> {
+		// TODO validate whether restricted property is allowed
+
 		if (!filter || typeof filter !== "object")
 			throw new Error(
 				"Invalid filter provided. Filter must be an object."
@@ -614,48 +616,116 @@ export default class DataModule extends BaseModule {
 			} else {
 				// Here we handle any normal keys in the query object
 
+				let currentKey = key;
+
 				// If the key doesn't exist in the schema, throw an error
-				if (!Object.hasOwn(schema, key))
-					throw new Error(
-						`Key "${key} does not exist in the schema."`
-					);
+				if (!Object.hasOwn(schema, key)) {
+					if (key.indexOf(".") !== -1) {
+						currentKey = key.substring(0, key.indexOf("."));
+
+						if (!Object.hasOwn(schema, currentKey))
+							throw new Error(
+								`Key "${currentKey}" does not exist in the schema.`
+							);
+
+						if (
+							schema[currentKey].type !== Types.Schema &&
+							(schema[currentKey].type !== Types.Array ||
+								(schema[currentKey].item.type !==
+									Types.Schema &&
+									schema[currentKey].item.type !==
+										Types.Array))
+						)
+							throw new Error(
+								`Key "${currentKey}" is not a schema/array.`
+							);
+					} else
+						throw new Error(
+							`Key "${key}" does not exist in the schema.`
+						);
+				}
 
 				// If the key in the schema is marked as restricted, containsRestrictedProperties will be true
-				if (schema[key].restricted) containsRestrictedProperties = true;
+				if (schema[currentKey].restricted)
+					containsRestrictedProperties = true;
 
 				// Handle schema type
-				if (schema[key].type === Types.Schema) {
+				if (schema[currentKey].type === Types.Schema) {
+					let subFilter;
+					if (key.indexOf(".") !== -1) {
+						const subKey = key.substring(
+							key.indexOf(".") + 1,
+							key.length
+						);
+						subFilter = {
+							[subKey]: value
+						};
+					} else subFilter = value;
+
 					// Run parseFindFilter on the nested schema object
 					const {
 						mongoFilter: _mongoFilter,
 						containsRestrictedProperties:
 							_containsRestrictedProperties
 					} = await this.parseFindFilter(
-						value,
-						schema[key].schema,
+						subFilter,
+						schema[currentKey].schema,
 						options
 					);
-					mongoFilter[key] = _mongoFilter;
+					mongoFilter[currentKey] = _mongoFilter;
 					if (_containsRestrictedProperties)
 						containsRestrictedProperties = true;
 				}
 				// Handle array type
-				else if (schema[key].type === Types.Array) {
-					// // Run parseFindFilter on the nested schema object
-					// const {
-					// 	mongoFilter: _mongoFilter,
-					// 	containsRestrictedProperties:
-					// 		_containsRestrictedProperties
-					// } = await this.parseFindFilter(
-					// 	value,
-					// 	schema[key].schema,
-					// 	options
-					// );
-					// mongoFilter[key] = _mongoFilter;
-					// if (_containsRestrictedProperties)
-					// 	containsRestrictedProperties = true;
-
-					throw new Error("NOT SUPPORTED YET.");
+				else if (schema[currentKey].type === Types.Array) {
+					const isNullOrUndefined =
+						value === null || value === undefined;
+					if (isNullOrUndefined)
+						throw new Error(
+							`Value for key ${currentKey} is an array item, so it cannot be null/undefined.`
+						);
+
+					// The type of the array items
+					const itemType = schema[currentKey].item.type;
+
+					// Handle nested arrays, which are not supported
+					if (itemType === Types.Array)
+						throw new Error("Nested arrays not supported");
+					// Handle schema array item type
+					else if (itemType === Types.Schema) {
+						let subFilter;
+						if (key.indexOf(".") !== -1) {
+							const subKey = key.substring(
+								key.indexOf(".") + 1,
+								key.length
+							);
+							subFilter = {
+								[subKey]: value
+							};
+						} else subFilter = value;
+
+						const {
+							mongoFilter: _mongoFilter,
+							containsRestrictedProperties:
+								_containsRestrictedProperties
+						} = await this.parseFindFilter(
+							subFilter,
+							schema[currentKey].item.schema,
+							options
+						);
+						mongoFilter[currentKey] = _mongoFilter;
+						if (_containsRestrictedProperties)
+							containsRestrictedProperties = true;
+					}
+					// Normal array item type
+					else {
+						// TODO possibly handle if a user gives some weird value here, like an object or array or $ operator
+
+						mongoFilter[currentKey] = this.getCastedValue(
+							value,
+							itemType
+						);
+					}
 				}
 				// else if (
 				// 	operators &&