2 Commits 89b92a39e8 ... 63090ed872

Author SHA1 Message Date
  Owen Diffey 63090ed872 feat: Sync model indexes on data module startup 10 months ago
  Owen Diffey 5372d26694 feat: Add getData as schema plugin (WIP) 10 months ago

+ 24 - 1
backend/src/main.ts

@@ -46,9 +46,32 @@ global.rs = () => {
 
 setTimeout(async () => {
 	const Model = await jobQueue.runJob("data", "getModel", { name: "abc" });
-	console.log("Model", Model);
+	// console.log("Model", Model);
 	const abcs = await Model.find({});
 	console.log("Abcs", abcs);
+	console.log(
+		"getData",
+		await Model.getData({
+			page: 1,
+			pageSize: 3,
+			properties: [
+				"title",
+				"markdown",
+				"status",
+				"showToNewUsers",
+				"createdBy"
+			],
+			sort: {},
+			queries: [
+				{
+					data: "v7",
+					filter: { property: "title" },
+					filterType: "contains"
+				}
+			],
+			operator: "and"
+		})
+	);
 
 	Model.create({
 		name: "Test name",

+ 18 - 0
backend/src/modules/DataModule.ts

@@ -10,6 +10,7 @@ import JobContext from "../JobContext";
 import BaseModule, { ModuleStatus } from "../BaseModule";
 import { UniqueMethods } from "../types/Modules";
 import { Models, Schemas } from "../types/Models";
+import getDataPlugin from "../schemas/plugins/getData";
 
 export default class DataModule extends BaseModule {
 	private models?: Models;
@@ -51,8 +52,14 @@ export default class DataModule extends BaseModule {
 
 		mongoose.SchemaTypes.String.set("trim", true);
 
+		this.mongoConnection.plugin(getDataPlugin, {
+			tags: ["useGetDataPlugin"]
+		});
+
 		await this.loadModels();
 
+		await this.syncModelIndexes();
+
 		// @ts-ignore
 		//        this.redisClient = createClient({ ...config.get("redis") });
 		//
@@ -152,6 +159,17 @@ export default class DataModule extends BaseModule {
 		};
 	}
 
+	/**
+	 * syncModelIndexes - Sync indexes for all models
+	 */
+	private async syncModelIndexes() {
+		if (!this.models) throw new Error("Models not loaded");
+
+		await Promise.all(
+			Object.values(this.models).map(model => model.syncIndexes())
+		);
+	}
+
 	/**
 	 * getModel - Get model
 	 *

+ 2 - 1
backend/src/schemas/news.ts

@@ -90,6 +90,7 @@ export const schema = new Schema<
 				if (showToNewUsers) return query.where({ showToNewUsers });
 				return query;
 			}
-		}
+		},
+		pluginTags: ["useGetDataPlugin"]
 	}
 );

+ 238 - 0
backend/src/schemas/plugins/getData.ts

@@ -0,0 +1,238 @@
+import { PipelineStage, Schema, SchemaOptions } from "mongoose";
+
+export enum FilterType {
+	REGEX = "regex",
+	CONTAINS = "contains",
+	EXACT = "exact",
+	DATETIME_BEFORE = "datetimeBefore",
+	DATETIME_AFTER = "datetimeAfter",
+	NUMBER_LESSER_EQUAL = "numberLesserEqual",
+	NUMBER_LESSER = "numberLesser",
+	NUMBER_GREATER = "numberGreater",
+	NUMBER_GREATER_EQUAL = "numberGreaterEqual",
+	NUMBER_EQUAL = "numberEquals",
+	BOOLEAN = "boolean",
+	SPECIAL = "special"
+}
+
+export interface GetDataSchemaOptions extends SchemaOptions {
+	getData?: {
+		blacklistedProperties?: string[];
+		specialProperties?: Record<string, PipelineStage[]>;
+		specialQueries?: Record<
+			string,
+			(query: Record<string, any>) => Record<string, any>
+		>;
+		specialFilters?: Record<string, (...args: any[]) => PipelineStage[]>;
+	};
+}
+
+export default function getDataPlugin(
+	schema: Schema,
+	options: GetDataSchemaOptions
+) {
+	schema.static(
+		"getData",
+		async function getData(payload: {
+			page: number;
+			pageSize: number;
+			properties: string[];
+			sort: Record<string, "ascending" | "descending">;
+			queries: {
+				data: any;
+				filter: {
+					property: string;
+				};
+				filterType: FilterType;
+			}[];
+			operator: "and" | "or" | "nor";
+		}) {
+			const { page, pageSize, properties, sort, queries, operator } =
+				payload;
+
+			const {
+				blacklistedProperties,
+				specialFilters,
+				specialProperties,
+				specialQueries
+			} = options.getData ?? {};
+
+			const pipeline: PipelineStage[] = [];
+
+			// If a query filter property or sort property is blacklisted, throw error
+			if (Array.isArray(blacklistedProperties)) {
+				if (
+					queries.some(query =>
+						blacklistedProperties.some(blacklistedProperty =>
+							blacklistedProperty.startsWith(
+								query.filter.property
+							)
+						)
+					)
+				)
+					throw new Error(
+						"Unable to filter by blacklisted property."
+					);
+				if (
+					Object.keys(sort).some(property =>
+						blacklistedProperties.some(blacklistedProperty =>
+							blacklistedProperty.startsWith(property)
+						)
+					)
+				)
+					throw new Error("Unable to sort by blacklisted property.");
+			}
+
+			// If a filter or property exists for a special property, add some custom pipeline steps
+			if (typeof specialProperties === "object")
+				Object.entries(specialProperties).forEach(
+					([specialProperty, pipelineSteps]) => {
+						// Check if a filter with the special property exists
+						const filterExists =
+							queries
+								.map(query => query.filter.property)
+								.indexOf(specialProperty) !== -1;
+
+						// Check if a property with the special property exists
+						const propertyExists =
+							properties.indexOf(specialProperty) !== -1;
+
+						// If no such filter or property exists, skip this function
+						if (!filterExists && !propertyExists) return;
+
+						// Add the specified pipeline steps into the pipeline
+						pipeline.push(...pipelineSteps);
+					}
+				);
+
+			// Adds the match stage to aggregation pipeline, which is responsible for filtering
+			const filterQueries = queries.flatMap(query => {
+				const { data, filter, filterType } = query;
+				const { property } = filter;
+
+				const newQuery: any = {};
+				switch (filterType) {
+					case FilterType.REGEX:
+						newQuery[property] = new RegExp(
+							`${data.slice(1, data.length - 1)}`,
+							"i"
+						);
+						break;
+					case FilterType.CONTAINS:
+						newQuery[property] = new RegExp(
+							`${data.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&")}`,
+							"i"
+						);
+						break;
+					case FilterType.EXACT:
+						newQuery[property] = data.toString();
+						break;
+					case FilterType.DATETIME_BEFORE:
+						newQuery[property] = { $lte: new Date(data) };
+						break;
+					case FilterType.DATETIME_AFTER:
+						newQuery[property] = { $gte: new Date(data) };
+						break;
+					case FilterType.NUMBER_LESSER_EQUAL:
+						newQuery[property] = { $lte: Number(data) };
+						break;
+					case FilterType.NUMBER_LESSER:
+						newQuery[property] = { $lt: Number(data) };
+						break;
+					case FilterType.NUMBER_GREATER:
+						newQuery[property] = { $gt: Number(data) };
+						break;
+					case FilterType.NUMBER_GREATER_EQUAL:
+						newQuery[property] = { $gte: Number(data) };
+						break;
+					case FilterType.NUMBER_EQUAL:
+						newQuery[property] = { $eq: Number(data) };
+						break;
+					case FilterType.BOOLEAN:
+						newQuery[property] = { $eq: !!data };
+						break;
+					case FilterType.SPECIAL:
+						if (
+							typeof specialFilters === "object" &&
+							typeof specialFilters[filter.property] ===
+								"function"
+						) {
+							pipeline.push(
+								...specialFilters[filter.property](data)
+							);
+							newQuery[property] = { $eq: true };
+						}
+						break;
+					default:
+						throw new Error(`Invalid filter type for "${filter}"`);
+				}
+
+				if (
+					typeof specialQueries === "object" &&
+					typeof specialQueries[filter.property] === "function"
+				) {
+					return specialQueries[filter.property](newQuery);
+				}
+
+				return newQuery;
+			});
+
+			const filterQuery: any = {};
+
+			if (filterQueries.length > 0)
+				filterQuery[`$${operator}`] = filterQueries;
+
+			pipeline.push({ $match: filterQuery });
+
+			// Adds sort stage to aggregation pipeline if there is at least one column being sorted, responsible for sorting data
+			if (Object.keys(sort).length > 0)
+				pipeline.push({
+					$sort: Object.fromEntries(
+						Object.entries(sort).map(([property, direction]) => [
+							property,
+							direction === "ascending" ? 1 : -1
+						])
+					)
+				});
+
+			// Adds first project stage to aggregation pipeline, responsible for including only the requested properties
+			pipeline.push({
+				$project: Object.fromEntries(
+					properties.map(property => [property, 1])
+				)
+			});
+
+			// Adds second project stage to aggregation pipeline, responsible for excluding some specific properties
+			if (
+				Array.isArray(blacklistedProperties) &&
+				blacklistedProperties.length > 0
+			)
+				pipeline.push({
+					$project: Object.fromEntries(
+						blacklistedProperties.map(property => [property, 0])
+					)
+				});
+
+			// Adds the facet stage to aggregation pipeline, responsible for returning a total document count, skipping and limitting the documents that will be returned
+			pipeline.push({
+				$facet: {
+					count: [{ $count: "count" }],
+					documents: [
+						{ $skip: pageSize * (page - 1) },
+						{ $limit: pageSize }
+					]
+				}
+			});
+
+			// Executes the aggregation pipeline
+			const [result] = await this.aggregate(pipeline).exec();
+
+			if (result.count.length === 0) return { data: [], count: 0 };
+
+			const { documents: data } = result;
+			const { count } = result.count[0];
+
+			return { data, count };
+		}
+	);
+}

+ 102 - 99
backend/src/schemas/station.ts

@@ -60,111 +60,114 @@ export interface StationSchema {
 	};
 }
 
-export const schema = new Schema<StationSchema>({
-	type: {
-		type: SchemaTypes.String,
-		enum: Object.values(StationType),
-		required: true
-	},
-	name: {
-		type: SchemaTypes.String,
-		unique: true,
-		minLength: 2,
-		maxLength: 16,
-		required: true
-	},
-	displayName: {
-		type: SchemaTypes.String,
-		unique: true,
-		minLength: 2,
-		maxLength: 32,
-		required: true
-	},
-	description: {
-		type: SchemaTypes.String,
-		minLength: 2,
-		maxLength: 128,
-		required: true
-	},
-	privacy: {
-		type: SchemaTypes.String,
-		default: StationPrivacy.PRIVATE,
-		enum: Object.values(StationPrivacy),
-		required: true
-	},
-	theme: {
-		type: SchemaTypes.String,
-		default: StationTheme.BLUE,
-		enum: Object.values(StationTheme),
-		required: true
-	},
-	owner: {
-		type: SchemaTypes.ObjectId,
-		required: false
-	},
-	djs: [{ type: SchemaTypes.ObjectId }],
-	currentSong: {
-		type: SchemaTypes.ObjectId,
-		required: false
-	},
-	currentSongIndex: {
-		type: SchemaTypes.Number,
-		required: false
-	},
-	startedAt: {
-		type: SchemaTypes.Date,
-		required: false
-	},
-	paused: {
-		type: SchemaTypes.Boolean,
-		default: false
-	},
-	timePaused: {
-		type: SchemaTypes.Number,
-		default: 0
-	},
-	pausedAt: {
-		type: SchemaTypes.Date,
-		required: false
-	},
-	playlist: {
-		type: SchemaTypes.ObjectId
-	},
-	queue: [{ type: SchemaTypes.ObjectId }],
-	blacklist: [{ type: SchemaTypes.ObjectId }],
-	requests: {
-		enabled: {
-			type: SchemaTypes.Boolean,
-			default: true
+export const schema = new Schema<StationSchema>(
+	{
+		type: {
+			type: SchemaTypes.String,
+			enum: Object.values(StationType),
+			required: true
+		},
+		name: {
+			type: SchemaTypes.String,
+			unique: true,
+			minLength: 2,
+			maxLength: 16,
+			required: true
 		},
-		access: {
+		displayName: {
 			type: SchemaTypes.String,
-			default: StationRequestsAccess.OWNER,
-			enum: Object.values(StationRequestsAccess)
+			unique: true,
+			minLength: 2,
+			maxLength: 32,
+			required: true
 		},
-		limit: {
+		description: {
+			type: SchemaTypes.String,
+			minLength: 2,
+			maxLength: 128,
+			required: true
+		},
+		privacy: {
+			type: SchemaTypes.String,
+			default: StationPrivacy.PRIVATE,
+			enum: Object.values(StationPrivacy),
+			required: true
+		},
+		theme: {
+			type: SchemaTypes.String,
+			default: StationTheme.BLUE,
+			enum: Object.values(StationTheme),
+			required: true
+		},
+		owner: {
+			type: SchemaTypes.ObjectId,
+			required: false
+		},
+		djs: [{ type: SchemaTypes.ObjectId }],
+		currentSong: {
+			type: SchemaTypes.ObjectId,
+			required: false
+		},
+		currentSongIndex: {
 			type: SchemaTypes.Number,
-			default: 5,
-			min: 1,
-			max: 50
-		}
-	},
-	autofill: {
-		enabled: {
+			required: false
+		},
+		startedAt: {
+			type: SchemaTypes.Date,
+			required: false
+		},
+		paused: {
 			type: SchemaTypes.Boolean,
-			default: true
+			default: false
 		},
-		playlists: [{ type: SchemaTypes.ObjectId }],
-		limit: {
+		timePaused: {
 			type: SchemaTypes.Number,
-			default: 30,
-			min: 1,
-			max: 50
+			default: 0
 		},
-		mode: {
-			type: SchemaTypes.String,
-			default: StationAutofillMode.RANDOM,
-			enum: Object.values(StationAutofillMode)
+		pausedAt: {
+			type: SchemaTypes.Date,
+			required: false
+		},
+		playlist: {
+			type: SchemaTypes.ObjectId
+		},
+		queue: [{ type: SchemaTypes.ObjectId }],
+		blacklist: [{ type: SchemaTypes.ObjectId }],
+		requests: {
+			enabled: {
+				type: SchemaTypes.Boolean,
+				default: true
+			},
+			access: {
+				type: SchemaTypes.String,
+				default: StationRequestsAccess.OWNER,
+				enum: Object.values(StationRequestsAccess)
+			},
+			limit: {
+				type: SchemaTypes.Number,
+				default: 5,
+				min: 1,
+				max: 50
+			}
+		},
+		autofill: {
+			enabled: {
+				type: SchemaTypes.Boolean,
+				default: true
+			},
+			playlists: [{ type: SchemaTypes.ObjectId }],
+			limit: {
+				type: SchemaTypes.Number,
+				default: 30,
+				min: 1,
+				max: 50
+			},
+			mode: {
+				type: SchemaTypes.String,
+				default: StationAutofillMode.RANDOM,
+				enum: Object.values(StationAutofillMode)
+			}
 		}
-	}
-});
+	},
+	{ pluginTags: ["useGetDataPlugin"] }
+);

+ 13 - 7
backend/src/types/Models.ts

@@ -1,15 +1,21 @@
-import { Model, InferSchemaType, Schema } from "mongoose";
+import { Model, Schema } from "mongoose";
 import { AbcSchema } from "../schemas/abc";
-import { NewsSchema } from "../schemas/news";
+import { NewsQueryHelpers, NewsSchema } from "../schemas/news";
 import { StationSchema } from "../schemas/station";
 
 export type Schemas = {
 	abc: Schema<AbcSchema>;
-	news: Schema<NewsSchema>;
+	news: Schema<
+		NewsSchema,
+		Model<NewsSchema, NewsQueryHelpers>,
+		{},
+		NewsQueryHelpers
+	>;
 	station: Schema<StationSchema>;
 };
 
-export type Models = Record<
-	keyof Schemas,
-	Model<InferSchemaType<Schemas[keyof Schemas]>>
->;
+export type Models = {
+	abc: Model<AbcSchema>;
+	news: Model<NewsSchema, NewsQueryHelpers>;
+	station: Model<StationSchema>;
+};