|
|
@@ -1,54 +1,13 @@
|
|
|
import config from "config";
|
|
|
-import { Db, MongoClient, ObjectId } from "mongodb";
|
|
|
-import { createHash } from "node:crypto";
|
|
|
import { createClient, RedisClientType } from "redis";
|
|
|
+import mongoose from "mongoose";
|
|
|
import JobContext from "../JobContext";
|
|
|
import BaseModule from "../BaseModule";
|
|
|
-import Schema, { Types } from "../Schema";
|
|
|
-import { Collections } from "../types/Collections";
|
|
|
-import { Document as SchemaDocument } from "../types/Document";
|
|
|
import { UniqueMethods } from "../types/Modules";
|
|
|
-import { AttributeValue } from "../types/AttributeValue";
|
|
|
-
|
|
|
-type Entries<T> = {
|
|
|
- [K in keyof T]: [K, T[K]];
|
|
|
-}[keyof T][];
|
|
|
-
|
|
|
-interface ProjectionObject {
|
|
|
- [property: string]: boolean | string[] | ProjectionObject;
|
|
|
-}
|
|
|
-
|
|
|
-type Projection = null | undefined | string[] | ProjectionObject;
|
|
|
-
|
|
|
-type NormalizedProjection = {
|
|
|
- projection: [string, boolean][];
|
|
|
- mode: "includeAllBut" | "excludeAllBut";
|
|
|
-};
|
|
|
-
|
|
|
-interface MongoFilter {
|
|
|
- [property: string]:
|
|
|
- | AttributeValue
|
|
|
- | AttributeValue[]
|
|
|
- | MongoFilter
|
|
|
- | MongoFilter[];
|
|
|
-}
|
|
|
-
|
|
|
-interface Document {
|
|
|
- [property: string]:
|
|
|
- | AttributeValue
|
|
|
- | AttributeValue[]
|
|
|
- | Document
|
|
|
- | Document[];
|
|
|
-}
|
|
|
-
|
|
|
-type AllowedRestricted = boolean | string[] | null | undefined;
|
|
|
+import { Models, Schemas } from "../types/Models";
|
|
|
|
|
|
export default class DataModule extends BaseModule {
|
|
|
- private collections?: Collections;
|
|
|
-
|
|
|
- private mongoClient?: MongoClient;
|
|
|
-
|
|
|
- private mongoDb?: Db;
|
|
|
+ private models?: Models;
|
|
|
|
|
|
private redisClient?: RedisClientType;
|
|
|
|
|
|
@@ -67,11 +26,9 @@ export default class DataModule extends BaseModule {
|
|
|
|
|
|
const mongoUrl = config.get<string>("mongo.url");
|
|
|
|
|
|
- this.mongoClient = new MongoClient(mongoUrl);
|
|
|
- await this.mongoClient.connect();
|
|
|
- this.mongoDb = this.mongoClient.db();
|
|
|
+ await mongoose.connect(mongoUrl);
|
|
|
|
|
|
- await this.loadCollections();
|
|
|
+ await this.loadModels();
|
|
|
|
|
|
const { url } = config.get<{ url: string }>("redis");
|
|
|
|
|
|
@@ -108,1112 +65,48 @@ export default class DataModule extends BaseModule {
|
|
|
public override async shutdown() {
|
|
|
await super.shutdown();
|
|
|
if (this.redisClient) await this.redisClient.quit();
|
|
|
- if (this.mongoClient) await this.mongoClient.close(false);
|
|
|
+ await mongoose.disconnect();
|
|
|
}
|
|
|
|
|
|
|
|
|
- * loadColllection - Import and load collection schema
|
|
|
+ * loadModel - Import and load model schema
|
|
|
*
|
|
|
- * @param collectionName - Name of the collection
|
|
|
- * @returns Collection
|
|
|
+ * @param modelName - Name of the model
|
|
|
+ * @returns Model
|
|
|
*/
|
|
|
- private async loadCollection<T extends keyof Collections>(
|
|
|
- collectionName: T
|
|
|
+ private async loadModel<ModelName extends keyof Models>(
|
|
|
+ modelName: ModelName
|
|
|
) {
|
|
|
- const { default: schema }: { default: Schema } = await import(
|
|
|
- `../collections/${collectionName.toString()}`
|
|
|
+ const { schema }: { schema: Schemas[ModelName] } = await import(
|
|
|
+ `../models/${modelName.toString()}`
|
|
|
);
|
|
|
- return {
|
|
|
- schema,
|
|
|
- collection: this.mongoDb!.collection(collectionName.toString())
|
|
|
- };
|
|
|
+ return mongoose.model(modelName.toString(), schema);
|
|
|
}
|
|
|
|
|
|
|
|
|
- * loadCollections - Load and initialize all collections
|
|
|
+ * loadModels - Load and initialize all models
|
|
|
*
|
|
|
* @returns Promise
|
|
|
*/
|
|
|
- private async loadCollections() {
|
|
|
- this.collections = {
|
|
|
- abc: await this.loadCollection("abc"),
|
|
|
- station: await this.loadCollection("station")
|
|
|
+ private async loadModels() {
|
|
|
+ this.models = {
|
|
|
+ abc: await this.loadModel("abc"),
|
|
|
+ station: await this.loadModel("station")
|
|
|
};
|
|
|
}
|
|
|
|
|
|
|
|
|
- * Takes a raw projection and turns it into a projection we can easily use
|
|
|
+ * getModel - Get model
|
|
|
*
|
|
|
- * @param projection - The raw projection
|
|
|
- * @returns Normalized projection
|
|
|
+ * @param modelName - Name of the model
|
|
|
+ * @returns Model
|
|
|
*/
|
|
|
- private normalizeProjection(projection: Projection): NormalizedProjection {
|
|
|
- let initialProjection = projection;
|
|
|
- if (
|
|
|
- !(projection && typeof initialProjection === "object") &&
|
|
|
- !Array.isArray(initialProjection)
|
|
|
- )
|
|
|
- initialProjection = [];
|
|
|
-
|
|
|
-
|
|
|
- let flattenedProjection = this.flattenProjection(initialProjection);
|
|
|
-
|
|
|
-
|
|
|
- flattenedProjection = flattenedProjection.map(([key, value]) => {
|
|
|
- if (typeof value !== "boolean") return [key, !!value];
|
|
|
- return [key, value];
|
|
|
- });
|
|
|
-
|
|
|
-
|
|
|
- const projectionKeys = flattenedProjection.map(([key]) => key);
|
|
|
- const uniqueProjectionKeys = new Set(projectionKeys);
|
|
|
- if (uniqueProjectionKeys.size !== flattenedProjection.length)
|
|
|
- throw new Error("Path collision, non-unique key");
|
|
|
-
|
|
|
-
|
|
|
- projectionKeys.forEach(key => {
|
|
|
-
|
|
|
- if (key.indexOf(".") !== -1) {
|
|
|
-
|
|
|
- const recursivelyCheckForPathCollision = (
|
|
|
- keyToCheck: string
|
|
|
- ) => {
|
|
|
-
|
|
|
- const subKey = keyToCheck.substring(
|
|
|
- 0,
|
|
|
- keyToCheck.lastIndexOf(".")
|
|
|
- );
|
|
|
-
|
|
|
- if (projectionKeys.indexOf(subKey) !== -1)
|
|
|
- throw new Error(
|
|
|
- `Path collision! ${key} collides with ${subKey}`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (subKey.indexOf(".") !== -1)
|
|
|
- recursivelyCheckForPathCollision(subKey);
|
|
|
- };
|
|
|
-
|
|
|
- recursivelyCheckForPathCollision(key);
|
|
|
- }
|
|
|
- });
|
|
|
-
|
|
|
-
|
|
|
- const anyNonIdTrues = flattenedProjection.reduce(
|
|
|
- (anyTrues, [key, value]) => anyTrues || (value && key !== "_id"),
|
|
|
- false
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- let mode: "includeAllBut" | "excludeAllBut" = "includeAllBut";
|
|
|
-
|
|
|
-
|
|
|
- if (anyNonIdTrues) mode = "excludeAllBut";
|
|
|
-
|
|
|
- return { projection: flattenedProjection, mode };
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Flatten the projection we've given (which can be an array of an object) into an array with key/value pairs
|
|
|
- *
|
|
|
- * @param projection - Projection
|
|
|
- * @returns
|
|
|
- */
|
|
|
- private flattenProjection(projection: Projection): [string, boolean][] {
|
|
|
- let flattenedProjection: [
|
|
|
- string,
|
|
|
- boolean | string[] | ProjectionObject
|
|
|
- ][] = [];
|
|
|
-
|
|
|
- if (!projection) throw new Error("Projection can't be null");
|
|
|
-
|
|
|
-
|
|
|
- if (Array.isArray(projection))
|
|
|
- flattenedProjection = projection.map(key => [key, true]);
|
|
|
- else if (typeof projection === "object")
|
|
|
- flattenedProjection = Object.entries(projection);
|
|
|
-
|
|
|
-
|
|
|
- const newProjection: [string, boolean][] = flattenedProjection.reduce(
|
|
|
- (currentEntries: [string, boolean][], [key, value]) => {
|
|
|
- if (typeof value === "object") {
|
|
|
- let flattenedValue = this.flattenProjection(value);
|
|
|
- flattenedValue = flattenedValue.map(
|
|
|
- ([nextKey, nextValue]) => [
|
|
|
- `${key}.${nextKey}`,
|
|
|
- nextValue
|
|
|
- ]
|
|
|
- );
|
|
|
- return [...currentEntries, ...flattenedValue];
|
|
|
- }
|
|
|
- return [...currentEntries, [key, value]];
|
|
|
- },
|
|
|
- []
|
|
|
- );
|
|
|
-
|
|
|
- return newProjection;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Parse a projection based on the schema and any given projection
|
|
|
- * If no projection is given, it will exclude any restricted properties
|
|
|
- * If a projection is given, it will exclude restricted properties that are not explicitly allowed in a projection
|
|
|
- * It will return a projection used in Mongo, and if any restricted property is explicitly allowed, return that we can't use the cache
|
|
|
- *
|
|
|
- * @param schema - The schema object
|
|
|
- * @param projection - The project, which can be null
|
|
|
- * @returns
|
|
|
- */
|
|
|
- private async parseFindProjection(
|
|
|
- projection: NormalizedProjection,
|
|
|
- schema: SchemaDocument,
|
|
|
- allowedRestricted: AllowedRestricted
|
|
|
+ public getModel<ModelName extends keyof Models>(
|
|
|
+ jobContext: JobContext,
|
|
|
+ modelName: ModelName
|
|
|
) {
|
|
|
-
|
|
|
- const mongoProjection: ProjectionObject = {};
|
|
|
-
|
|
|
- let canCache = true;
|
|
|
-
|
|
|
- const unfilteredEntries = Object.entries(schema);
|
|
|
- await Promise.all(
|
|
|
- unfilteredEntries.map(async ([key, value]) => {
|
|
|
- const { restricted } = value;
|
|
|
-
|
|
|
-
|
|
|
- const allowedByRestricted =
|
|
|
- !restricted ||
|
|
|
- this.allowedByRestricted(allowedRestricted, key);
|
|
|
-
|
|
|
-
|
|
|
- if (allowedByRestricted && restricted) {
|
|
|
- canCache = false;
|
|
|
- }
|
|
|
-
|
|
|
- else if (!allowedByRestricted) {
|
|
|
- mongoProjection[key] = false;
|
|
|
- }
|
|
|
-
|
|
|
- else if (value.type === Types.Schema) {
|
|
|
-
|
|
|
- const deeperProjection = this.getDeeperProjection(
|
|
|
- projection,
|
|
|
- key
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- const deeperAllowedRestricted =
|
|
|
- this.getDeeperAllowedRestricted(allowedRestricted, key);
|
|
|
-
|
|
|
- if (!value.schema) throw new Error("Schema is not defined");
|
|
|
-
|
|
|
- const parsedProjection = await this.parseFindProjection(
|
|
|
- deeperProjection,
|
|
|
- value.schema,
|
|
|
- deeperAllowedRestricted
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- Object.keys(parsedProjection.mongoProjection).length > 0
|
|
|
- )
|
|
|
- mongoProjection[key] = parsedProjection.mongoProjection;
|
|
|
-
|
|
|
-
|
|
|
- canCache = canCache && parsedProjection.canCache;
|
|
|
- }
|
|
|
- })
|
|
|
- );
|
|
|
-
|
|
|
- return {
|
|
|
- canCache,
|
|
|
- mongoProjection
|
|
|
- };
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Whether a property is allowed if it's restricted
|
|
|
- *
|
|
|
- * @param projection - The projection object/array
|
|
|
- * @param property - Property name
|
|
|
- * @returns
|
|
|
- */
|
|
|
- private allowedByRestricted(
|
|
|
- allowedRestricted: AllowedRestricted,
|
|
|
- property: string
|
|
|
- ) {
|
|
|
-
|
|
|
- if (allowedRestricted === true) return true;
|
|
|
-
|
|
|
- if (!allowedRestricted) return false;
|
|
|
-
|
|
|
- if (!Array.isArray(allowedRestricted)) return false;
|
|
|
-
|
|
|
-
|
|
|
- if (allowedRestricted.indexOf(property) !== -1) return true;
|
|
|
-
|
|
|
-
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Whether a property is allowed in a projection array/object
|
|
|
- *
|
|
|
- * @param projection - The projection object/array
|
|
|
- * @param property - Property name
|
|
|
- * @returns
|
|
|
- */
|
|
|
- private allowedByProjection(
|
|
|
- projection: NormalizedProjection,
|
|
|
- property: string
|
|
|
- ) {
|
|
|
- const obj = Object.fromEntries(projection.projection);
|
|
|
-
|
|
|
- if (projection.mode === "excludeAllBut") {
|
|
|
-
|
|
|
- if (obj[property]) return true;
|
|
|
-
|
|
|
-
|
|
|
- const nestedTrue = projection.projection.reduce(
|
|
|
- (nestedTrue, [key, value]) => {
|
|
|
- if (value && key.startsWith(`${property}.`)) return true;
|
|
|
- return nestedTrue;
|
|
|
- },
|
|
|
- false
|
|
|
- );
|
|
|
-
|
|
|
- return nestedTrue;
|
|
|
- }
|
|
|
-
|
|
|
- if (projection.mode === "includeAllBut") {
|
|
|
-
|
|
|
- if (obj[property] === false) return false;
|
|
|
-
|
|
|
-
|
|
|
- return true;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Returns the projection array/object that is one level deeper based on the property key
|
|
|
- *
|
|
|
- * @param projection - The projection object/array
|
|
|
- * @param key - The property key
|
|
|
- * @returns Array or Object
|
|
|
- */
|
|
|
- private getDeeperProjection(
|
|
|
- projection: NormalizedProjection,
|
|
|
- currentKey: string
|
|
|
- ): NormalizedProjection {
|
|
|
- const newProjection: [string, boolean][] = projection.projection
|
|
|
-
|
|
|
- .map(([key, value]) => {
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- key.indexOf(".") === -1 ||
|
|
|
- !key.startsWith(`${currentKey}.`)
|
|
|
- )
|
|
|
- return false;
|
|
|
-
|
|
|
- const lowerKey = key.substring(
|
|
|
- key.indexOf(".") + 1,
|
|
|
- key.length
|
|
|
- );
|
|
|
-
|
|
|
- if (lowerKey.length === 0) return false;
|
|
|
- return [lowerKey, value];
|
|
|
- })
|
|
|
-
|
|
|
-
|
|
|
- .filter((entries): entries is [string, boolean] => !!entries);
|
|
|
-
|
|
|
-
|
|
|
- return { projection: newProjection, mode: projection.mode };
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Returns the allowedRestricted that is one level deeper based on the property key
|
|
|
- *
|
|
|
- * @param projection - The projection object/array
|
|
|
- * @param key - The property key
|
|
|
- * @returns Array or Object
|
|
|
- */
|
|
|
- private getDeeperAllowedRestricted(
|
|
|
- allowedRestricted: AllowedRestricted,
|
|
|
- currentKey: string
|
|
|
- ): AllowedRestricted {
|
|
|
-
|
|
|
- if (typeof allowedRestricted === "boolean") return allowedRestricted;
|
|
|
- if (!Array.isArray(allowedRestricted)) return false;
|
|
|
-
|
|
|
- const newAllowedRestricted: string[] = <string[]>allowedRestricted
|
|
|
-
|
|
|
- .map(key => {
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- key.indexOf(".") === -1 ||
|
|
|
- !key.startsWith(`${currentKey}.`)
|
|
|
- )
|
|
|
- return false;
|
|
|
-
|
|
|
- const lowerKey = key.substring(
|
|
|
- key.indexOf(".") + 1,
|
|
|
- key.length
|
|
|
- );
|
|
|
-
|
|
|
- if (lowerKey.length === 0) return false;
|
|
|
- return lowerKey;
|
|
|
- })
|
|
|
-
|
|
|
- .filter(entries => entries);
|
|
|
-
|
|
|
-
|
|
|
- return newAllowedRestricted;
|
|
|
- }
|
|
|
-
|
|
|
- private getCastedValue(value: unknown, schemaType: Types): AttributeValue {
|
|
|
- if (value === null || value === undefined) return null;
|
|
|
-
|
|
|
- if (schemaType === Types.String) {
|
|
|
-
|
|
|
- const castedValue =
|
|
|
- typeof value === "string" ? value : String(value);
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- }
|
|
|
- if (schemaType === Types.Number) {
|
|
|
-
|
|
|
- const castedValue =
|
|
|
- typeof value === "number" ? value : Number(value);
|
|
|
-
|
|
|
- if (Number.isNaN(castedValue))
|
|
|
- throw new Error(
|
|
|
- `Cast error, number cannot be NaN, with value ${value}`
|
|
|
- );
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- }
|
|
|
- if (schemaType === Types.Date) {
|
|
|
-
|
|
|
- const castedValue =
|
|
|
- Object.prototype.toString.call(value) === "[object Date]"
|
|
|
- ? (value as Date)
|
|
|
- : new Date(value.toString());
|
|
|
-
|
|
|
- if (new Date(castedValue).toString() === "Invalid Date")
|
|
|
- throw new Error(
|
|
|
- `Cast error, date cannot be invalid, with value ${value}`
|
|
|
- );
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- }
|
|
|
- if (schemaType === Types.Boolean) {
|
|
|
-
|
|
|
- const castedValue =
|
|
|
- typeof value === "boolean" ? value : Boolean(value);
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- }
|
|
|
- if (schemaType === Types.ObjectId) {
|
|
|
- if (typeof value !== "string" && !(value instanceof ObjectId))
|
|
|
- throw new Error(
|
|
|
- `Cast error, ObjectId invalid, with value ${value}`
|
|
|
- );
|
|
|
-
|
|
|
- const castedValue = new ObjectId(value);
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- }
|
|
|
- throw new Error(
|
|
|
- `Unsupported schema type found with type ${Types[schemaType]}. This should never happen.`
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * parseFindFilter - Ensure validity of filter and return a mongo filter
|
|
|
- *
|
|
|
- * @param filter - Filter
|
|
|
- * @param schema - Schema of collection document
|
|
|
- * @param options - Parser options
|
|
|
- * @returns Promise returning object with query values cast to schema types
|
|
|
- * and whether query includes restricted attributes
|
|
|
- */
|
|
|
- private async parseFindFilter(
|
|
|
- filter: MongoFilter,
|
|
|
- schema: SchemaDocument,
|
|
|
- allowedRestricted: AllowedRestricted,
|
|
|
- options?: {
|
|
|
- operators?: boolean;
|
|
|
- }
|
|
|
- ): Promise<{
|
|
|
- mongoFilter: MongoFilter;
|
|
|
- containsRestrictedProperties: boolean;
|
|
|
- canCache: boolean;
|
|
|
- }> {
|
|
|
- if (!filter || typeof filter !== "object")
|
|
|
- throw new Error(
|
|
|
- "Invalid filter provided. Filter must be an object."
|
|
|
- );
|
|
|
-
|
|
|
- const keys = Object.keys(filter);
|
|
|
- if (keys.length === 0)
|
|
|
- throw new Error(
|
|
|
- "Invalid filter provided. Filter must contain keys."
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- const operators = !(options && options.operators === false);
|
|
|
-
|
|
|
- const mongoFilter: MongoFilter = {};
|
|
|
-
|
|
|
- let containsRestrictedProperties = false;
|
|
|
-
|
|
|
- let canCache = true;
|
|
|
-
|
|
|
-
|
|
|
- const allowedKeyOperators = ["$or", "$and"];
|
|
|
-
|
|
|
- const allowedValueOperators = ["$in"];
|
|
|
-
|
|
|
-
|
|
|
- await Promise.all(
|
|
|
- Object.entries(filter).map(async ([key, value]) => {
|
|
|
-
|
|
|
- if (!key || key.length === 0)
|
|
|
- throw new Error(
|
|
|
- `Invalid filter provided. Key must be at least 1 character.`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (operators && key[0] === "$") {
|
|
|
-
|
|
|
- if (allowedKeyOperators.indexOf(key) === -1)
|
|
|
- throw new Error(
|
|
|
- `Invalid filter provided. Operator "${key}" is not allowed.`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (key === "$or" || key === "$and") {
|
|
|
-
|
|
|
- if (!Array.isArray(value) || value.length === 0)
|
|
|
- throw new Error(
|
|
|
- `Key "${key}" must contain array of filters.`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- mongoFilter[key] = [];
|
|
|
-
|
|
|
-
|
|
|
- await Promise.all(
|
|
|
- value.map(async _value => {
|
|
|
-
|
|
|
- if (
|
|
|
- !_value ||
|
|
|
- typeof _value !== "object" ||
|
|
|
- _value.constructor.name !== "Object"
|
|
|
- )
|
|
|
- throw Error("not an object");
|
|
|
-
|
|
|
- const {
|
|
|
- mongoFilter: _mongoFilter,
|
|
|
- containsRestrictedProperties:
|
|
|
- _containsRestrictedProperties
|
|
|
- } = await this.parseFindFilter(
|
|
|
- _value as MongoFilter,
|
|
|
- schema,
|
|
|
- allowedRestricted,
|
|
|
- options
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- (<MongoFilter[]>mongoFilter[key]).push(
|
|
|
- _mongoFilter
|
|
|
- );
|
|
|
- if (_containsRestrictedProperties)
|
|
|
- containsRestrictedProperties = true;
|
|
|
- })
|
|
|
- );
|
|
|
- } else
|
|
|
- throw new Error(
|
|
|
- `Unhandled operator "${key}", this should never happen!`
|
|
|
- );
|
|
|
- } else {
|
|
|
-
|
|
|
-
|
|
|
- let currentKey = key;
|
|
|
-
|
|
|
-
|
|
|
- if (!Object.hasOwn(schema, key)) {
|
|
|
- if (key.indexOf(".") !== -1) {
|
|
|
- currentKey = key.substring(0, key.indexOf("."));
|
|
|
-
|
|
|
- if (!Object.hasOwn(schema, currentKey))
|
|
|
- throw new Error(
|
|
|
- `Key "${currentKey}" does not exist in the schema.`
|
|
|
- );
|
|
|
-
|
|
|
- if (
|
|
|
- schema[currentKey].type !== Types.Schema &&
|
|
|
- (schema[currentKey].type !== Types.Array ||
|
|
|
- (schema[currentKey].item!.type !==
|
|
|
- Types.Schema &&
|
|
|
- schema[currentKey].item!.type !==
|
|
|
- Types.Array))
|
|
|
- )
|
|
|
- throw new Error(
|
|
|
- `Key "${currentKey}" is not a schema/array.`
|
|
|
- );
|
|
|
- } else
|
|
|
- throw new Error(
|
|
|
- `Key "${key}" does not exist in the schema.`
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- const { restricted } = schema[currentKey];
|
|
|
-
|
|
|
-
|
|
|
- const allowedByRestricted =
|
|
|
- !restricted ||
|
|
|
- this.allowedByRestricted(allowedRestricted, currentKey);
|
|
|
-
|
|
|
- if (!allowedByRestricted)
|
|
|
- throw new Error(`Key "${currentKey}" is restricted.`);
|
|
|
-
|
|
|
-
|
|
|
- if (restricted) containsRestrictedProperties = true;
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- operators &&
|
|
|
- typeof value === "object" &&
|
|
|
- value &&
|
|
|
- Object.keys(value).length === 1 &&
|
|
|
- Object.keys(value)[0] &&
|
|
|
- Object.keys(value)[0][0] === "$"
|
|
|
- ) {
|
|
|
-
|
|
|
- const operator = Object.keys(value)[0];
|
|
|
-
|
|
|
-
|
|
|
- if (allowedValueOperators.indexOf(operator) === -1)
|
|
|
- throw new Error(
|
|
|
- `Invalid filter provided. Operator "${operator}" is not allowed.`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (operator === "$in") {
|
|
|
-
|
|
|
- let { type } = schema[currentKey];
|
|
|
-
|
|
|
- if (type === Types.Schema)
|
|
|
- throw new Error(
|
|
|
- `Key "${currentKey}" is of type schema, which is not allowed with $in`
|
|
|
- );
|
|
|
-
|
|
|
- if (type === Types.Array)
|
|
|
- type = schema[key].item!.type;
|
|
|
-
|
|
|
- const value$in = (<{ $in: AttributeValue[] }>value)
|
|
|
- .$in;
|
|
|
- let filter$in: AttributeValue[] = [];
|
|
|
-
|
|
|
- if (!Array.isArray(value$in))
|
|
|
- throw new Error("$in musr be array");
|
|
|
-
|
|
|
-
|
|
|
- if (value$in.length > 0)
|
|
|
- filter$in = await Promise.all(
|
|
|
- value$in.map(async _value => {
|
|
|
- const isNullOrUndefined =
|
|
|
- _value === null ||
|
|
|
- _value === undefined;
|
|
|
- if (isNullOrUndefined)
|
|
|
- throw new Error(
|
|
|
- `Value for key ${currentKey} using $in is undefuned/null, which is not allowed.`
|
|
|
- );
|
|
|
-
|
|
|
- const castedValue = this.getCastedValue(
|
|
|
- _value,
|
|
|
- type
|
|
|
- );
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- })
|
|
|
- );
|
|
|
-
|
|
|
- mongoFilter[currentKey] = { $in: filter$in };
|
|
|
- } else
|
|
|
- throw new Error(
|
|
|
- `Unhandled operator "${operator}", this should never happen!`
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- else if (schema[currentKey].type === Types.Schema) {
|
|
|
- let subFilter;
|
|
|
- if (key.indexOf(".") !== -1) {
|
|
|
- const subKey = key.substring(
|
|
|
- key.indexOf(".") + 1,
|
|
|
- key.length
|
|
|
- );
|
|
|
- subFilter = {
|
|
|
- [subKey]: value
|
|
|
- };
|
|
|
- } else subFilter = value;
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- !subFilter ||
|
|
|
- typeof subFilter !== "object" ||
|
|
|
- subFilter.constructor.name !== "Object"
|
|
|
- )
|
|
|
- throw Error("not an object");
|
|
|
-
|
|
|
-
|
|
|
- const deeperAllowedRestricted =
|
|
|
- this.getDeeperAllowedRestricted(
|
|
|
- allowedRestricted,
|
|
|
- currentKey
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- const {
|
|
|
- mongoFilter: _mongoFilter,
|
|
|
- containsRestrictedProperties:
|
|
|
- _containsRestrictedProperties
|
|
|
- } = await this.parseFindFilter(
|
|
|
- subFilter as MongoFilter,
|
|
|
- schema[currentKey].schema!,
|
|
|
- deeperAllowedRestricted,
|
|
|
- options
|
|
|
- );
|
|
|
- mongoFilter[currentKey] = _mongoFilter;
|
|
|
- if (_containsRestrictedProperties)
|
|
|
- containsRestrictedProperties = true;
|
|
|
- }
|
|
|
-
|
|
|
- else if (schema[currentKey].type === Types.Array) {
|
|
|
- const isNullOrUndefined =
|
|
|
- value === null || value === undefined;
|
|
|
- if (isNullOrUndefined)
|
|
|
- throw new Error(
|
|
|
- `Value for key ${currentKey} is an array item, so it cannot be null/undefined.`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- const itemType = schema[currentKey].item!.type;
|
|
|
-
|
|
|
-
|
|
|
- if (itemType === Types.Array)
|
|
|
- throw new Error("Nested arrays not supported");
|
|
|
-
|
|
|
- else if (itemType === Types.Schema) {
|
|
|
- let subFilter;
|
|
|
- if (key.indexOf(".") !== -1) {
|
|
|
- const subKey = key.substring(
|
|
|
- key.indexOf(".") + 1,
|
|
|
- key.length
|
|
|
- );
|
|
|
- subFilter = {
|
|
|
- [subKey]: value
|
|
|
- };
|
|
|
- } else subFilter = value;
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- typeof subFilter !== "object" ||
|
|
|
- subFilter.constructor.name !== "Object"
|
|
|
- )
|
|
|
- throw Error("not an object");
|
|
|
-
|
|
|
-
|
|
|
- const deeperAllowedRestricted =
|
|
|
- this.getDeeperAllowedRestricted(
|
|
|
- allowedRestricted,
|
|
|
- currentKey
|
|
|
- );
|
|
|
-
|
|
|
- const {
|
|
|
- mongoFilter: _mongoFilter,
|
|
|
- containsRestrictedProperties:
|
|
|
- _containsRestrictedProperties
|
|
|
- } = await this.parseFindFilter(
|
|
|
- subFilter as MongoFilter,
|
|
|
- schema[currentKey].item!.schema!,
|
|
|
- deeperAllowedRestricted,
|
|
|
- options
|
|
|
- );
|
|
|
- mongoFilter[currentKey] = _mongoFilter;
|
|
|
- if (_containsRestrictedProperties)
|
|
|
- containsRestrictedProperties = true;
|
|
|
- }
|
|
|
-
|
|
|
- else {
|
|
|
-
|
|
|
- if (Array.isArray(value)) throw Error("an array");
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- typeof value === "object" &&
|
|
|
- value.constructor.name === "Object"
|
|
|
- )
|
|
|
- throw Error("an object");
|
|
|
-
|
|
|
- mongoFilter[currentKey] = this.getCastedValue(
|
|
|
- value as AttributeValue,
|
|
|
- itemType
|
|
|
- );
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- else {
|
|
|
- const isNullOrUndefined =
|
|
|
- value === null || value === undefined;
|
|
|
- if (isNullOrUndefined && schema[key].required)
|
|
|
- throw new Error(
|
|
|
- `Value for key ${key} is required, so it cannot be null/undefined.`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (isNullOrUndefined) mongoFilter[key] = null;
|
|
|
-
|
|
|
- else {
|
|
|
- const schemaType = schema[key].type;
|
|
|
-
|
|
|
-
|
|
|
- if (Array.isArray(value)) throw Error("an array");
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- typeof value === "object" &&
|
|
|
- value.constructor.name === "Object"
|
|
|
- )
|
|
|
- throw Error("an object");
|
|
|
-
|
|
|
- mongoFilter[key] = this.getCastedValue(
|
|
|
- value as AttributeValue,
|
|
|
- schemaType
|
|
|
- );
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- })
|
|
|
- );
|
|
|
-
|
|
|
- if (containsRestrictedProperties) canCache = false;
|
|
|
-
|
|
|
- return { mongoFilter, containsRestrictedProperties, canCache };
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * Strip a document object from any unneeded properties, or of any restricted properties
|
|
|
- * If a projection is given
|
|
|
- * Also casts some values
|
|
|
- *
|
|
|
- * @param document - The document object
|
|
|
- * @param schema - The schema object
|
|
|
- * @param projection - The projection, which can be null
|
|
|
- */
|
|
|
- private async stripDocument(
|
|
|
- document: Document,
|
|
|
- schema: SchemaDocument,
|
|
|
- projection: NormalizedProjection,
|
|
|
- allowedRestricted: AllowedRestricted
|
|
|
- ): Promise<Document> {
|
|
|
- const unfilteredEntries = Object.entries(document);
|
|
|
-
|
|
|
- const filteredEntries: Entries<Document> = [];
|
|
|
- await Promise.all(
|
|
|
- unfilteredEntries.map(async ([key, value]) => {
|
|
|
-
|
|
|
- if (!schema[key]) return;
|
|
|
-
|
|
|
-
|
|
|
- const allowedByProjection = this.allowedByProjection(
|
|
|
- projection,
|
|
|
- key
|
|
|
- );
|
|
|
-
|
|
|
- const allowedByRestricted =
|
|
|
- !schema[key].restricted ||
|
|
|
- this.allowedByRestricted(allowedRestricted, key);
|
|
|
-
|
|
|
- if (!allowedByProjection) return;
|
|
|
- if (!allowedByRestricted) return;
|
|
|
-
|
|
|
-
|
|
|
- if (schema[key].type === Types.Schema) {
|
|
|
-
|
|
|
- if (!value) {
|
|
|
- filteredEntries.push([key, null]);
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- if (
|
|
|
- typeof value !== "object" ||
|
|
|
- value.constructor.name !== "Object"
|
|
|
- )
|
|
|
- throw Error("not an object");
|
|
|
-
|
|
|
-
|
|
|
- const deeperProjection = this.getDeeperProjection(
|
|
|
- projection,
|
|
|
- key
|
|
|
- );
|
|
|
-
|
|
|
- const deeperAllowedRestricted =
|
|
|
- this.getDeeperAllowedRestricted(allowedRestricted, key);
|
|
|
-
|
|
|
-
|
|
|
- const strippedDocument = await this.stripDocument(
|
|
|
- value as Document,
|
|
|
- schema[key].schema!,
|
|
|
- deeperProjection,
|
|
|
- deeperAllowedRestricted
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (Object.keys(strippedDocument).length > 0) {
|
|
|
- filteredEntries.push([key, strippedDocument]);
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- filteredEntries.push([key, {}]);
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- if (schema[key].type === Types.Array) {
|
|
|
-
|
|
|
- if (!value) {
|
|
|
- filteredEntries.push([key, null]);
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- if (!Array.isArray(value)) throw Error("not an array");
|
|
|
-
|
|
|
-
|
|
|
- const itemType = schema[key].item!.type;
|
|
|
-
|
|
|
- const items = (await Promise.all(
|
|
|
- value.map(async item => {
|
|
|
-
|
|
|
- if (itemType === Types.Schema) {
|
|
|
-
|
|
|
- if (
|
|
|
- !item ||
|
|
|
- typeof item !== "object" ||
|
|
|
- item.constructor.name !== "Object"
|
|
|
- )
|
|
|
- throw Error("not an object");
|
|
|
-
|
|
|
-
|
|
|
- const deeperProjection =
|
|
|
- this.getDeeperProjection(projection, key);
|
|
|
-
|
|
|
- const deeperAllowedRestricted =
|
|
|
- this.getDeeperAllowedRestricted(
|
|
|
- allowedRestricted,
|
|
|
- key
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- const strippedDocument =
|
|
|
- await this.stripDocument(
|
|
|
- item as Document,
|
|
|
- schema[key].item!.schema!,
|
|
|
- deeperProjection,
|
|
|
- deeperAllowedRestricted
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- if (Object.keys(strippedDocument).length > 0)
|
|
|
- return strippedDocument;
|
|
|
-
|
|
|
-
|
|
|
- return {};
|
|
|
- }
|
|
|
-
|
|
|
- if (itemType === Types.Array) {
|
|
|
- throw new Error("Nested arrays not supported");
|
|
|
- }
|
|
|
-
|
|
|
- else {
|
|
|
-
|
|
|
- const isNullOrUndefined =
|
|
|
- item === null || item === undefined;
|
|
|
- if (isNullOrUndefined) return null;
|
|
|
-
|
|
|
-
|
|
|
- const castedValue = this.getCastedValue(
|
|
|
- item,
|
|
|
- itemType
|
|
|
- );
|
|
|
-
|
|
|
- return castedValue;
|
|
|
- }
|
|
|
- })
|
|
|
- )) as AttributeValue[] | Document[];
|
|
|
-
|
|
|
- filteredEntries.push([key, items]);
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- const castedValue = this.getCastedValue(
|
|
|
- value,
|
|
|
- schema[key].type
|
|
|
- );
|
|
|
-
|
|
|
- filteredEntries.push([key, castedValue]);
|
|
|
- })
|
|
|
- );
|
|
|
-
|
|
|
- return Object.fromEntries(filteredEntries);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- * find - Get one or more document(s) from a single collection
|
|
|
- *
|
|
|
- * @param payload - Payload
|
|
|
- * @returns Returned object
|
|
|
- */
|
|
|
- public async find<CollectionNameType extends keyof Collections>(
|
|
|
- context: JobContext,
|
|
|
- {
|
|
|
- collection,
|
|
|
- filter,
|
|
|
- projection,
|
|
|
- allowedRestricted,
|
|
|
- limit = 1,
|
|
|
- page = 1,
|
|
|
- useCache = true
|
|
|
- }: {
|
|
|
- collection: CollectionNameType;
|
|
|
- filter: MongoFilter;
|
|
|
- projection?: Projection;
|
|
|
- allowedRestricted?: boolean | string[];
|
|
|
- limit?: number;
|
|
|
- page?: number;
|
|
|
- useCache?: boolean;
|
|
|
- }
|
|
|
- ) {
|
|
|
-
|
|
|
- if (page < 1) throw new Error("Page must be at least 1");
|
|
|
- if (limit < 1) throw new Error("Limit must be at least 1");
|
|
|
- if (limit > 100) throw new Error("Limit must not be greater than 100");
|
|
|
-
|
|
|
-
|
|
|
- if (!collection) throw new Error("No collection specified");
|
|
|
- if (this.collections && !this.collections[collection])
|
|
|
- throw new Error("Collection not found");
|
|
|
-
|
|
|
- const { schema } = this.collections![collection];
|
|
|
-
|
|
|
-
|
|
|
- const normalizedProjection = this.normalizeProjection(projection);
|
|
|
-
|
|
|
-
|
|
|
- const parsedProjection = await this.parseFindProjection(
|
|
|
- normalizedProjection,
|
|
|
- schema.getDocument(),
|
|
|
- allowedRestricted
|
|
|
- );
|
|
|
-
|
|
|
- let cacheable = useCache !== false && parsedProjection.canCache;
|
|
|
- const { mongoProjection } = parsedProjection;
|
|
|
-
|
|
|
-
|
|
|
- const parsedFilter = await this.parseFindFilter(
|
|
|
- filter,
|
|
|
- schema.getDocument(),
|
|
|
- allowedRestricted
|
|
|
- );
|
|
|
-
|
|
|
- cacheable = cacheable && parsedFilter.canCache;
|
|
|
- const { mongoFilter } = parsedFilter;
|
|
|
- let queryHash: string | null = null;
|
|
|
- let documents: Document[] | null = null;
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- if (cacheable) {
|
|
|
-
|
|
|
- queryHash = createHash("md5")
|
|
|
- .update(
|
|
|
- JSON.stringify({
|
|
|
- collection,
|
|
|
- mongoFilter,
|
|
|
- limit,
|
|
|
- page
|
|
|
- })
|
|
|
- )
|
|
|
- .digest("hex");
|
|
|
-
|
|
|
-
|
|
|
- const cachedQuery = await this.redisClient?.GET(
|
|
|
- `query.find.${queryHash}`
|
|
|
- );
|
|
|
-
|
|
|
-
|
|
|
- documents = cachedQuery ? JSON.parse(cachedQuery) : null;
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- if (documents) {
|
|
|
- cacheable = false;
|
|
|
- } else {
|
|
|
- const totalCount = await this.collections?.[
|
|
|
- collection
|
|
|
- ].collection.countDocuments(mongoFilter);
|
|
|
- if (totalCount === 0 || totalCount === undefined)
|
|
|
- return limit === 1 ? null : [];
|
|
|
- const lastPage = Math.ceil(totalCount / limit);
|
|
|
- if (lastPage < page)
|
|
|
- throw new Error(`The last page available is ${lastPage}`);
|
|
|
-
|
|
|
-
|
|
|
- documents = (await this.collections?.[collection].collection
|
|
|
- .find(mongoFilter, mongoProjection)
|
|
|
- .limit(limit)
|
|
|
- .skip((page - 1) * limit)
|
|
|
- .toArray()) as Document[];
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- if (cacheable && queryHash) {
|
|
|
- this.redisClient!.SET(
|
|
|
- `query.find.${queryHash}`,
|
|
|
- JSON.stringify(documents),
|
|
|
- {
|
|
|
- EX: 60
|
|
|
- }
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- documents = await Promise.all(
|
|
|
- documents.map(async (document: Document) =>
|
|
|
- this.stripDocument(
|
|
|
- document,
|
|
|
- schema.getDocument(),
|
|
|
- normalizedProjection,
|
|
|
- allowedRestricted
|
|
|
- )
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- if (!documents || documents!.length === 0)
|
|
|
- return limit === 1 ? null : [];
|
|
|
- return limit === 1 ? documents![0] : documents;
|
|
|
+ if (!this.models) throw new Error("Models not loaded");
|
|
|
+ return this.models[modelName];
|
|
|
}
|
|
|
}
|
|
|
|
Owen Diffey