index.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. import config from "config";
  2. import mongoose from "mongoose";
  3. import bluebird from "bluebird";
  4. import async from "async";
  5. import CoreClass from "../../core";
  6. const REQUIRED_DOCUMENT_VERSIONS = {
  7. activity: 2,
  8. news: 3,
  9. playlist: 6,
  10. punishment: 1,
  11. queueSong: 1,
  12. report: 6,
  13. song: 9,
  14. station: 8,
  15. user: 4,
  16. youtubeApiRequest: 1,
  17. youtubeVideo: 1,
  18. ratings: 1,
  19. importJob: 1
  20. };
  21. const regex = {
  22. azAZ09_: /^[A-Za-z0-9_]+$/,
  23. az09_: /^[a-z0-9_]+$/,
  24. emailSimple: /^[\x00-\x7F]+@[a-z0-9]+\.[a-z0-9]+(\.[a-z0-9]+)?$/,
  25. ascii: /^[\x00-\x7F]+$/,
  26. name: /^[\p{L}0-9 .'_-]+$/u,
  27. custom: regex => new RegExp(`^[${regex}]+$`)
  28. };
  29. const isLength = (string, min, max) => !(typeof string !== "string" || string.length < min || string.length > max);
  30. mongoose.Promise = bluebird;
  31. let DBModule;
  32. class _DBModule extends CoreClass {
  33. // eslint-disable-next-line require-jsdoc
  34. constructor() {
  35. super("db");
  36. DBModule = this;
  37. }
  38. /**
  39. * Initialises the database module
  40. *
  41. * @returns {Promise} - returns promise (reject, resolve)
  42. */
  43. initialize() {
  44. return new Promise((resolve, reject) => {
  45. this.schemas = {};
  46. this.models = {};
  47. const mongoUrl = config.get("mongo").url;
  48. mongoose
  49. .connect(mongoUrl, {
  50. useNewUrlParser: true,
  51. useUnifiedTopology: true
  52. })
  53. .then(async () => {
  54. this.schemas = {
  55. song: {},
  56. queueSong: {},
  57. station: {},
  58. user: {},
  59. dataRequest: {},
  60. activity: {},
  61. playlist: {},
  62. news: {},
  63. report: {},
  64. punishment: {},
  65. youtubeApiRequest: {},
  66. youtubeVideo: {},
  67. ratings: {}
  68. };
  69. const importSchema = schemaName =>
  70. new Promise(resolve => {
  71. import(`./schemas/${schemaName}`).then(schema => {
  72. this.schemas[schemaName] = new mongoose.Schema(schema.default);
  73. return resolve();
  74. });
  75. });
  76. await importSchema("song");
  77. await importSchema("queueSong");
  78. await importSchema("station");
  79. await importSchema("user");
  80. await importSchema("dataRequest");
  81. await importSchema("activity");
  82. await importSchema("playlist");
  83. await importSchema("news");
  84. await importSchema("report");
  85. await importSchema("punishment");
  86. await importSchema("youtubeApiRequest");
  87. await importSchema("youtubeVideo");
  88. await importSchema("ratings");
  89. await importSchema("importJob");
  90. this.models = {
  91. song: mongoose.model("song", this.schemas.song),
  92. queueSong: mongoose.model("queueSong", this.schemas.queueSong),
  93. station: mongoose.model("station", this.schemas.station),
  94. user: mongoose.model("user", this.schemas.user),
  95. dataRequest: mongoose.model("dataRequest", this.schemas.dataRequest),
  96. activity: mongoose.model("activity", this.schemas.activity),
  97. playlist: mongoose.model("playlist", this.schemas.playlist),
  98. news: mongoose.model("news", this.schemas.news),
  99. report: mongoose.model("report", this.schemas.report),
  100. punishment: mongoose.model("punishment", this.schemas.punishment),
  101. youtubeApiRequest: mongoose.model("youtubeApiRequest", this.schemas.youtubeApiRequest),
  102. youtubeVideo: mongoose.model("youtubeVideo", this.schemas.youtubeVideo),
  103. ratings: mongoose.model("ratings", this.schemas.ratings),
  104. importJob: mongoose.model("importJob", this.schemas.importJob)
  105. };
  106. mongoose.connection.on("error", err => {
  107. this.log("ERROR", err);
  108. });
  109. mongoose.connection.on("disconnected", () => {
  110. this.log("ERROR", "Disconnected, going to try to reconnect...");
  111. this.setStatus("RECONNECTING");
  112. });
  113. mongoose.connection.on("reconnected", () => {
  114. this.log("INFO", "Reconnected.");
  115. this.setStatus("READY");
  116. });
  117. mongoose.connection.on("reconnectFailed", () => {
  118. this.log("INFO", "Reconnect failed, stopping reconnecting.");
  119. this.setStatus("FAILED");
  120. });
  121. // User
  122. this.schemas.user
  123. .path("username")
  124. .validate(
  125. username =>
  126. isLength(username, 2, 32) &&
  127. regex.custom("a-zA-Z0-9_-").test(username) &&
  128. username.replaceAll(/[_]/g, "").length > 0,
  129. "Invalid username."
  130. );
  131. this.schemas.user.path("email.address").validate(email => {
  132. if (!isLength(email, 3, 254)) return false;
  133. if (email.indexOf("@") !== email.lastIndexOf("@")) return false;
  134. return regex.emailSimple.test(email) && regex.ascii.test(email);
  135. }, "Invalid email.");
  136. this.schemas.user
  137. .path("name")
  138. .validate(
  139. name =>
  140. isLength(name, 1, 64) &&
  141. regex.name.test(name) &&
  142. name.replaceAll(/[ .'_-]/g, "").length > 0,
  143. "Invalid name."
  144. );
  145. // Station
  146. this.schemas.station
  147. .path("name")
  148. .validate(id => isLength(id, 2, 16) && regex.az09_.test(id), "Invalid station name.");
  149. this.schemas.station
  150. .path("displayName")
  151. .validate(
  152. displayName => isLength(displayName, 2, 32) && regex.ascii.test(displayName),
  153. "Invalid display name."
  154. );
  155. this.schemas.station.path("description").validate(description => {
  156. if (!isLength(description, 2, 200)) return false;
  157. const characters = description.split("");
  158. return characters.filter(character => character.charCodeAt(0) === 21328).length === 0;
  159. }, "Invalid display name.");
  160. this.schemas.station.path("owner").validate({
  161. validator: owner =>
  162. new Promise((resolve, reject) => {
  163. this.models.station.countDocuments({ owner }, (err, c) => {
  164. if (err) reject(new Error("A mongo error happened."));
  165. else if (c >= 25) reject(new Error("User already has 25 stations."));
  166. else resolve();
  167. });
  168. }),
  169. message: "User already has 25 stations."
  170. });
  171. // Song
  172. const songTitle = title => isLength(title, 1, 100);
  173. this.schemas.song.path("title").validate(songTitle, "Invalid title.");
  174. this.schemas.song.path("artists").validate(artists => artists.length <= 10, "Invalid artists.");
  175. const songArtists = artists =>
  176. artists.filter(artist => isLength(artist, 1, 64) && artist !== "NONE").length ===
  177. artists.length;
  178. this.schemas.song.path("artists").validate(songArtists, "Invalid artists.");
  179. const songGenres = genres => {
  180. if (genres.length > 16) return false;
  181. return (
  182. genres.filter(genre => isLength(genre, 1, 32) && regex.ascii.test(genre)).length ===
  183. genres.length
  184. );
  185. };
  186. this.schemas.song.path("genres").validate(songGenres, "Invalid genres.");
  187. const songTags = tags =>
  188. tags.filter(tag => /^[a-zA-Z0-9_]{1,64}$|^[a-zA-Z0-9_]{1,64}\[[a-zA-Z0-9_]{1,64}\]$/.test(tag))
  189. .length === tags.length;
  190. this.schemas.song.path("tags").validate(songTags, "Invalid tags.");
  191. const songThumbnail = thumbnail => {
  192. if (!isLength(thumbnail, 1, 256)) return false;
  193. if (config.get("cookie.secure") === true) return thumbnail.startsWith("https://");
  194. return thumbnail.startsWith("http://") || thumbnail.startsWith("https://");
  195. };
  196. this.schemas.song.path("thumbnail").validate(songThumbnail, "Invalid thumbnail.");
  197. // Playlist
  198. this.schemas.playlist
  199. .path("displayName")
  200. .validate(
  201. displayName => isLength(displayName, 1, 32) && regex.ascii.test(displayName),
  202. "Invalid display name."
  203. );
  204. this.schemas.playlist.path("createdBy").validate(createdBy => {
  205. this.models.playlist.countDocuments({ createdBy }, (err, c) => !(err || c >= 100));
  206. }, "Max 100 playlists per user.");
  207. this.schemas.playlist
  208. .path("songs")
  209. .validate(songs => songs.length <= 10000, "Max 10000 songs per playlist.");
  210. // this.schemas.playlist.path("songs").validate(songs => {
  211. // if (songs.length === 0) return true;
  212. // return songs[0].duration <= 10800;
  213. // }, "Max 3 hours per song.");
  214. this.models.activity.syncIndexes();
  215. this.models.dataRequest.syncIndexes();
  216. this.models.news.syncIndexes();
  217. this.models.playlist.syncIndexes();
  218. this.models.punishment.syncIndexes();
  219. this.models.queueSong.syncIndexes();
  220. this.models.report.syncIndexes();
  221. this.models.song.syncIndexes();
  222. this.models.station.syncIndexes();
  223. this.models.user.syncIndexes();
  224. this.models.youtubeApiRequest.syncIndexes();
  225. this.models.youtubeVideo.syncIndexes();
  226. this.models.ratings.syncIndexes();
  227. this.models.importJob.syncIndexes();
  228. if (config.get("skipDbDocumentsVersionCheck")) resolve();
  229. else {
  230. this.runJob("CHECK_DOCUMENT_VERSIONS", {}, null, -1)
  231. .then(() => {
  232. resolve();
  233. })
  234. .catch(err => {
  235. reject(err);
  236. });
  237. }
  238. })
  239. .catch(err => {
  240. this.log("ERROR", err);
  241. reject(err);
  242. });
  243. });
  244. }
  245. /**
  246. * Checks if all documents have the correct document version
  247. *
  248. * @returns {Promise} - returns promise (reject, resolve)
  249. */
  250. CHECK_DOCUMENT_VERSIONS() {
  251. return new Promise((resolve, reject) => {
  252. async.each(
  253. Object.keys(REQUIRED_DOCUMENT_VERSIONS),
  254. (modelName, next) => {
  255. const model = DBModule.models[modelName];
  256. const requiredDocumentVersion = REQUIRED_DOCUMENT_VERSIONS[modelName];
  257. model.countDocuments({ documentVersion: { $ne: requiredDocumentVersion } }, (err, count) => {
  258. if (err) next(err);
  259. else if (count > 0)
  260. next(
  261. `Collection "${modelName}" has ${count} documents with a wrong document version. Run migration.`
  262. );
  263. else next();
  264. });
  265. },
  266. err => {
  267. if (err) reject(new Error(err));
  268. else resolve();
  269. }
  270. );
  271. });
  272. }
  273. /**
  274. * Returns a database model
  275. *
  276. * @param {object} payload - object containing the payload
  277. * @param {object} payload.modelName - name of the model to get
  278. * @returns {Promise} - returns promise (reject, resolve)
  279. */
  280. GET_MODEL(payload) {
  281. return new Promise(resolve => {
  282. resolve(DBModule.models[payload.modelName]);
  283. });
  284. }
  285. /**
  286. * Returns a database schema
  287. *
  288. * @param {object} payload - object containing the payload
  289. * @param {object} payload.schemaName - name of the schema to get
  290. * @returns {Promise} - returns promise (reject, resolve)
  291. */
  292. GET_SCHEMA(payload) {
  293. return new Promise(resolve => {
  294. resolve(DBModule.schemas[payload.schemaName]);
  295. });
  296. }
  297. /**
  298. * Gets data
  299. *
  300. * @param {object} payload - object containing the payload
  301. * @param {string} payload.page - the page
  302. * @param {string} payload.pageSize - the page size
  303. * @param {string} payload.properties - the properties to return for each song
  304. * @param {string} payload.sort - the sort object
  305. * @param {string} payload.queries - the queries array
  306. * @param {string} payload.operator - the operator for queries
  307. * @param {string} payload.modelName - the db collection modal name
  308. * @param {string} payload.blacklistedProperties - the properties that are not allowed to be returned, filtered by or sorted by
  309. * @param {string} payload.specialProperties - the special properties
  310. * @param {string} payload.specialQueries - the special queries
  311. * @returns {Promise} - returns a promise (resolve, reject)
  312. */
  313. GET_DATA(payload) {
  314. return new Promise((resolve, reject) => {
  315. async.waterfall(
  316. [
  317. // Creates pipeline array
  318. next => next(null, []),
  319. // If a query filter property or sort property is blacklisted, throw error
  320. (pipeline, next) => {
  321. const { sort, queries, blacklistedProperties } = payload;
  322. if (
  323. queries.some(query =>
  324. blacklistedProperties.some(blacklistedProperty =>
  325. blacklistedProperty.startsWith(query.filter.property)
  326. )
  327. )
  328. )
  329. return next("Unable to filter by blacklisted property.");
  330. if (
  331. Object.keys(sort).some(property =>
  332. blacklistedProperties.some(blacklistedProperty =>
  333. blacklistedProperty.startsWith(property)
  334. )
  335. )
  336. )
  337. return next("Unable to sort by blacklisted property.");
  338. return next(null, pipeline);
  339. },
  340. // If a filter or property exists for a special property, add some custom pipeline steps
  341. (pipeline, next) => {
  342. const { properties, queries, specialProperties } = payload;
  343. async.eachLimit(
  344. Object.entries(specialProperties),
  345. 1,
  346. ([specialProperty, pipelineSteps], next) => {
  347. // Check if a filter with the special property exists
  348. const filterExists =
  349. queries.map(query => query.filter.property).indexOf(specialProperty) !== -1;
  350. // Check if a property with the special property exists
  351. const propertyExists = properties.indexOf(specialProperty) !== -1;
  352. // If no such filter or property exists, skip this function
  353. if (!filterExists && !propertyExists) return next();
  354. // Add the specified pipeline steps into the pipeline
  355. pipeline.push(...pipelineSteps);
  356. return next();
  357. },
  358. err => {
  359. next(err, pipeline);
  360. }
  361. );
  362. },
  363. // Adds the match stage to aggregation pipeline, which is responsible for filtering
  364. (pipeline, next) => {
  365. const { queries, operator, specialQueries, specialFilters } = payload;
  366. let queryError;
  367. const newQueries = queries.flatMap(query => {
  368. const { data, filter, filterType } = query;
  369. const newQuery = {};
  370. if (filterType === "regex") {
  371. newQuery[filter.property] = new RegExp(`${data.slice(1, data.length - 1)}`, "i");
  372. } else if (filterType === "contains") {
  373. newQuery[filter.property] = new RegExp(
  374. `${data.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&")}`,
  375. "i"
  376. );
  377. } else if (filterType === "exact") {
  378. newQuery[filter.property] = data.toString();
  379. } else if (filterType === "datetimeBefore") {
  380. newQuery[filter.property] = { $lte: new Date(data) };
  381. } else if (filterType === "datetimeAfter") {
  382. newQuery[filter.property] = { $gte: new Date(data) };
  383. } else if (filterType === "numberLesserEqual") {
  384. newQuery[filter.property] = { $lte: Number(data) };
  385. } else if (filterType === "numberLesser") {
  386. newQuery[filter.property] = { $lt: Number(data) };
  387. } else if (filterType === "numberGreater") {
  388. newQuery[filter.property] = { $gt: Number(data) };
  389. } else if (filterType === "numberGreaterEqual") {
  390. newQuery[filter.property] = { $gte: Number(data) };
  391. } else if (filterType === "numberEquals") {
  392. newQuery[filter.property] = { $eq: Number(data) };
  393. } else if (filterType === "boolean") {
  394. newQuery[filter.property] = { $eq: !!data };
  395. } else if (filterType === "special") {
  396. pipeline.push(...specialFilters[filter.property](data));
  397. newQuery[filter.property] = { $eq: true };
  398. }
  399. if (specialQueries[filter.property]) {
  400. return specialQueries[filter.property](newQuery);
  401. }
  402. return newQuery;
  403. });
  404. if (queryError) next(queryError);
  405. const queryObject = {};
  406. if (newQueries.length > 0) {
  407. if (operator === "and") queryObject.$and = newQueries;
  408. else if (operator === "or") queryObject.$or = newQueries;
  409. else if (operator === "nor") queryObject.$nor = newQueries;
  410. }
  411. pipeline.push({ $match: queryObject });
  412. next(null, pipeline);
  413. },
  414. // Adds sort stage to aggregation pipeline if there is at least one column being sorted, responsible for sorting data
  415. (pipeline, next) => {
  416. const { sort } = payload;
  417. const newSort = Object.fromEntries(
  418. Object.entries(sort).map(([property, direction]) => [
  419. property,
  420. direction === "ascending" ? 1 : -1
  421. ])
  422. );
  423. if (Object.keys(newSort).length > 0) pipeline.push({ $sort: newSort });
  424. next(null, pipeline);
  425. },
  426. // Adds first project stage to aggregation pipeline, responsible for including only the requested properties
  427. (pipeline, next) => {
  428. const { properties } = payload;
  429. pipeline.push({ $project: Object.fromEntries(properties.map(property => [property, 1])) });
  430. next(null, pipeline);
  431. },
  432. // Adds second project stage to aggregation pipeline, responsible for excluding some specific properties
  433. (pipeline, next) => {
  434. const { blacklistedProperties } = payload;
  435. if (blacklistedProperties.length > 0)
  436. pipeline.push({
  437. $project: Object.fromEntries(blacklistedProperties.map(property => [property, 0]))
  438. });
  439. next(null, pipeline);
  440. },
  441. // Adds the facet stage to aggregation pipeline, responsible for returning a total document count, skipping and limitting the documents that will be returned
  442. (pipeline, next) => {
  443. const { page, pageSize } = payload;
  444. pipeline.push({
  445. $facet: {
  446. count: [{ $count: "count" }],
  447. documents: [{ $skip: pageSize * (page - 1) }, { $limit: pageSize }]
  448. }
  449. });
  450. // console.dir(pipeline, { depth: 6 });
  451. next(null, pipeline);
  452. },
  453. (pipeline, next) => {
  454. const { modelName } = payload;
  455. DBModule.runJob("GET_MODEL", { modelName }, this)
  456. .then(model => {
  457. if (!model) return next("Invalid model.");
  458. return next(null, pipeline, model);
  459. })
  460. .catch(err => {
  461. next(err);
  462. });
  463. },
  464. // Executes the aggregation pipeline
  465. (pipeline, model, next) => {
  466. model.aggregate(pipeline).exec((err, result) => {
  467. // console.dir(err);
  468. // console.dir(result, { depth: 6 });
  469. if (err) return next(err);
  470. if (result[0].count.length === 0) return next(null, 0, []);
  471. const { count } = result[0].count[0];
  472. const { documents } = result[0];
  473. // console.log(111, err, result, count, documents[0]);
  474. return next(null, count, documents);
  475. });
  476. }
  477. ],
  478. (err, count, documents) => {
  479. if (err && err !== true) return reject(new Error(err));
  480. return resolve({ data: documents, count });
  481. }
  482. );
  483. });
  484. }
  485. /**
  486. * Checks if a password to be stored in the database has a valid length
  487. *
  488. * @param {object} password - the password itself
  489. * @returns {Promise} - returns promise (reject, resolve)
  490. */
  491. passwordValid(password) {
  492. return isLength(password, 6, 200);
  493. }
  494. }
  495. export default new _DBModule();