index.js 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630
  1. import config from "config";
  2. import mongoose from "mongoose";
  3. import bluebird from "bluebird";
  4. import async from "async";
  5. import CoreClass from "../../core";
  6. const REQUIRED_DOCUMENT_VERSIONS = {
  7. activity: 3,
  8. news: 3,
  9. playlist: 7,
  10. punishment: 1,
  11. queueSong: 1,
  12. report: 7,
  13. song: 10,
  14. station: 10,
  15. user: 4,
  16. youtubeApiRequest: 1,
  17. youtubeVideo: [1, 2],
  18. youtubeChannel: 1,
  19. ratings: 2,
  20. importJob: 1,
  21. stationHistory: 2,
  22. soundcloudTrack: 1,
  23. spotifyTrack: 1,
  24. spotifyAlbum: 1,
  25. spotifyArtist: 1,
  26. genericApiRequest: 1
  27. };
  28. const regex = {
  29. azAZ09_: /^[A-Za-z0-9_]+$/,
  30. az09_: /^[a-z0-9_]+$/,
  31. emailSimple: /^[\x00-\x7F]+@[a-z0-9]+\.[a-z0-9]+(\.[a-z0-9]+)?$/,
  32. ascii: /^[\x00-\x7F]+$/,
  33. name: /^[\p{L}0-9 .'_-]+$/u,
  34. custom: regex => new RegExp(`^[${regex}]+$`)
  35. };
  36. const isLength = (string, min, max) => !(typeof string !== "string" || string.length < min || string.length > max);
  37. mongoose.Promise = bluebird;
  38. let DBModule;
  39. class _DBModule extends CoreClass {
  40. // eslint-disable-next-line require-jsdoc
  41. constructor() {
  42. super("db");
  43. DBModule = this;
  44. }
  45. /**
  46. * Initialises the database module
  47. *
  48. * @returns {Promise} - returns promise (reject, resolve)
  49. */
  50. initialize() {
  51. return new Promise((resolve, reject) => {
  52. this.schemas = {};
  53. this.models = {};
  54. const { user, password, host, port, database } = config.get("mongo");
  55. mongoose
  56. .connect(`mongodb://${user}:${password}@${host}:${port}/${database}`, {
  57. useNewUrlParser: true,
  58. useUnifiedTopology: true
  59. })
  60. .then(async () => {
  61. this.schemas = {
  62. song: {},
  63. queueSong: {},
  64. station: {},
  65. user: {},
  66. dataRequest: {},
  67. activity: {},
  68. playlist: {},
  69. news: {},
  70. report: {},
  71. punishment: {},
  72. youtubeApiRequest: {},
  73. youtubeVideo: {},
  74. youtubeChannel: {},
  75. ratings: {},
  76. stationHistory: {},
  77. soundcloudTrack: {},
  78. spotifyTrack: {},
  79. spotifyAlbum: {},
  80. spotifyArtist: {},
  81. genericApiRequest: {}
  82. };
  83. const importSchema = schemaName =>
  84. new Promise(resolve => {
  85. import(`./schemas/${schemaName}`).then(schema => {
  86. this.schemas[schemaName] = new mongoose.Schema(schema.default);
  87. return resolve();
  88. });
  89. });
  90. await importSchema("song");
  91. await importSchema("queueSong");
  92. await importSchema("station");
  93. await importSchema("user");
  94. await importSchema("dataRequest");
  95. await importSchema("activity");
  96. await importSchema("playlist");
  97. await importSchema("news");
  98. await importSchema("report");
  99. await importSchema("punishment");
  100. await importSchema("youtubeApiRequest");
  101. await importSchema("youtubeVideo");
  102. await importSchema("youtubeChannel");
  103. await importSchema("ratings");
  104. await importSchema("importJob");
  105. await importSchema("stationHistory");
  106. await importSchema("soundcloudTrack");
  107. await importSchema("spotifyTrack");
  108. await importSchema("spotifyAlbum");
  109. await importSchema("spotifyArtist");
  110. await importSchema("genericApiRequest");
  111. this.models = {
  112. song: mongoose.model("song", this.schemas.song),
  113. queueSong: mongoose.model("queueSong", this.schemas.queueSong),
  114. station: mongoose.model("station", this.schemas.station),
  115. user: mongoose.model("user", this.schemas.user),
  116. dataRequest: mongoose.model("dataRequest", this.schemas.dataRequest),
  117. activity: mongoose.model("activity", this.schemas.activity),
  118. playlist: mongoose.model("playlist", this.schemas.playlist),
  119. news: mongoose.model("news", this.schemas.news),
  120. report: mongoose.model("report", this.schemas.report),
  121. punishment: mongoose.model("punishment", this.schemas.punishment),
  122. youtubeApiRequest: mongoose.model("youtubeApiRequest", this.schemas.youtubeApiRequest),
  123. youtubeVideo: mongoose.model("youtubeVideo", this.schemas.youtubeVideo),
  124. youtubeChannel: mongoose.model("youtubeChannel", this.schemas.youtubeChannel),
  125. ratings: mongoose.model("ratings", this.schemas.ratings),
  126. importJob: mongoose.model("importJob", this.schemas.importJob),
  127. stationHistory: mongoose.model("stationHistory", this.schemas.stationHistory),
  128. soundcloudTrack: mongoose.model("soundcloudTrack", this.schemas.soundcloudTrack),
  129. spotifyTrack: mongoose.model("spotifyTrack", this.schemas.spotifyTrack),
  130. spotifyAlbum: mongoose.model("spotifyAlbum", this.schemas.spotifyAlbum),
  131. spotifyArtist: mongoose.model("spotifyArtist", this.schemas.spotifyArtist),
  132. genericApiRequest: mongoose.model("genericApiRequest", this.schemas.genericApiRequest)
  133. };
  134. mongoose.connection.on("error", err => {
  135. this.log("ERROR", err);
  136. });
  137. mongoose.connection.on("disconnected", () => {
  138. this.log("ERROR", "Disconnected, going to try to reconnect...");
  139. this.setStatus("RECONNECTING");
  140. });
  141. mongoose.connection.on("reconnected", () => {
  142. this.log("INFO", "Reconnected.");
  143. this.setStatus("READY");
  144. });
  145. mongoose.connection.on("reconnectFailed", () => {
  146. this.log("INFO", "Reconnect failed, stopping reconnecting.");
  147. this.setStatus("FAILED");
  148. });
  149. // User
  150. this.schemas.user
  151. .path("username")
  152. .validate(
  153. username =>
  154. isLength(username, 2, 32) &&
  155. regex.custom("a-zA-Z0-9_-").test(username) &&
  156. username.replaceAll(/[_]/g, "").length > 0,
  157. "Invalid username."
  158. );
  159. this.schemas.user.path("email.address").validate(email => {
  160. if (!isLength(email, 3, 254)) return false;
  161. if (email.indexOf("@") !== email.lastIndexOf("@")) return false;
  162. return regex.emailSimple.test(email) && regex.ascii.test(email);
  163. }, "Invalid email.");
  164. this.schemas.user
  165. .path("name")
  166. .validate(
  167. name =>
  168. isLength(name, 1, 64) &&
  169. regex.name.test(name) &&
  170. name.replaceAll(/[ .'_-]/g, "").length > 0,
  171. "Invalid name."
  172. );
  173. // Station
  174. this.schemas.station
  175. .path("name")
  176. .validate(id => isLength(id, 2, 16) && regex.az09_.test(id), "Invalid station name.");
  177. this.schemas.station
  178. .path("displayName")
  179. .validate(
  180. displayName => isLength(displayName, 2, 32) && regex.ascii.test(displayName),
  181. "Invalid display name."
  182. );
  183. this.schemas.station.path("description").validate(description => {
  184. if (!isLength(description, 2, 200)) return false;
  185. const characters = description.split("");
  186. return characters.filter(character => character.charCodeAt(0) === 21328).length === 0;
  187. }, "Invalid display name.");
  188. this.schemas.station.path("owner").validate({
  189. validator: owner =>
  190. new Promise((resolve, reject) => {
  191. this.models.station.countDocuments({ owner }, (err, c) => {
  192. if (err) reject(new Error("A mongo error happened."));
  193. else if (c >= 25) reject(new Error("User already has 25 stations."));
  194. else resolve();
  195. });
  196. }),
  197. message: "User already has 25 stations."
  198. });
  199. this.schemas.station
  200. .path("requests.autorequestLimit")
  201. .validate(function validateRequestsAutorequestLimit(autorequestLimit) {
  202. const { limit } = this.get("requests");
  203. if (autorequestLimit > limit) return false;
  204. return true;
  205. }, "Autorequest limit cannot be higher than the request limit.");
  206. // Song
  207. this.schemas.song.path("mediaSource").validate(mediaSource => {
  208. if (mediaSource.startsWith("youtube:")) return true;
  209. if (mediaSource.startsWith("soundcloud:")) return true;
  210. if (mediaSource.startsWith("spotify:")) return true;
  211. return false;
  212. });
  213. const songTitle = title => isLength(title, 1, 100);
  214. this.schemas.song.path("title").validate(songTitle, "Invalid title.");
  215. this.schemas.song.path("artists").validate(artists => artists.length <= 10, "Invalid artists.");
  216. const songArtists = artists =>
  217. artists.filter(artist => isLength(artist, 1, 64) && artist !== "NONE").length ===
  218. artists.length;
  219. this.schemas.song.path("artists").validate(songArtists, "Invalid artists.");
  220. const songGenres = genres => {
  221. if (genres.length > 16) return false;
  222. return (
  223. genres.filter(genre => isLength(genre, 1, 32) && regex.ascii.test(genre)).length ===
  224. genres.length
  225. );
  226. };
  227. this.schemas.song.path("genres").validate(songGenres, "Invalid genres.");
  228. const songTags = tags =>
  229. tags.filter(tag => /^[a-zA-Z0-9_]{1,64}$|^[a-zA-Z0-9_]{1,64}\[[a-zA-Z0-9_]{1,64}\]$/.test(tag))
  230. .length === tags.length;
  231. this.schemas.song.path("tags").validate(songTags, "Invalid tags.");
  232. const songThumbnail = thumbnail => {
  233. if (!isLength(thumbnail, 1, 256)) return false;
  234. if (config.get("url.secure") === true) return thumbnail.startsWith("https://");
  235. return thumbnail.startsWith("http://") || thumbnail.startsWith("https://");
  236. };
  237. this.schemas.song.path("thumbnail").validate(songThumbnail, "Invalid thumbnail.");
  238. // Playlist
  239. this.schemas.playlist
  240. .path("displayName")
  241. .validate(
  242. displayName => isLength(displayName, 1, 32) && regex.ascii.test(displayName),
  243. "Invalid display name."
  244. );
  245. this.schemas.playlist.path("createdBy").validate(createdBy => {
  246. this.models.playlist.countDocuments({ createdBy }, (err, c) => !(err || c >= 100));
  247. }, "Max 100 playlists per user.");
  248. this.schemas.playlist
  249. .path("songs")
  250. .validate(songs => songs.length <= 10000, "Max 10000 songs per playlist.");
  251. // this.schemas.playlist.path("songs").validate(songs => {
  252. // if (songs.length === 0) return true;
  253. // return songs[0].duration <= 10800;
  254. // }, "Max 3 hours per song.");
  255. this.models.activity.syncIndexes();
  256. this.models.dataRequest.syncIndexes();
  257. this.models.news.syncIndexes();
  258. this.models.playlist.syncIndexes();
  259. this.models.punishment.syncIndexes();
  260. this.models.queueSong.syncIndexes();
  261. this.models.report.syncIndexes();
  262. this.models.song.syncIndexes();
  263. this.models.station.syncIndexes();
  264. this.models.user.syncIndexes();
  265. this.models.youtubeApiRequest.syncIndexes();
  266. this.models.youtubeVideo.syncIndexes();
  267. this.models.youtubeChannel.syncIndexes();
  268. this.models.ratings.syncIndexes();
  269. this.models.importJob.syncIndexes();
  270. this.models.stationHistory.syncIndexes();
  271. this.models.soundcloudTrack.syncIndexes();
  272. this.models.spotifyTrack.syncIndexes();
  273. this.models.spotifyArtist.syncIndexes();
  274. this.models.genericApiRequest.syncIndexes();
  275. if (config.get("skipDbDocumentsVersionCheck")) resolve();
  276. else {
  277. this.runJob("CHECK_DOCUMENT_VERSIONS", {}, null, -1)
  278. .then(() => {
  279. resolve();
  280. })
  281. .catch(err => {
  282. reject(err);
  283. });
  284. }
  285. })
  286. .catch(err => {
  287. this.log("ERROR", err);
  288. reject(err);
  289. });
  290. });
  291. }
  292. /**
  293. * Checks if all documents have the correct document version
  294. *
  295. * @returns {Promise} - returns promise (reject, resolve)
  296. */
  297. CHECK_DOCUMENT_VERSIONS() {
  298. return new Promise((resolve, reject) => {
  299. async.each(
  300. Object.keys(REQUIRED_DOCUMENT_VERSIONS),
  301. async modelName => {
  302. const model = DBModule.models[modelName];
  303. const requiredDocumentVersion = REQUIRED_DOCUMENT_VERSIONS[modelName];
  304. const count = await model.countDocuments({
  305. documentVersion: {
  306. $nin: Array.isArray(requiredDocumentVersion)
  307. ? requiredDocumentVersion
  308. : [requiredDocumentVersion]
  309. }
  310. });
  311. if (count > 0)
  312. throw new Error(
  313. `Collection "${modelName}" has ${count} documents with a wrong document version. Run migration.`
  314. );
  315. if (Array.isArray(requiredDocumentVersion)) {
  316. const count2 = await model.countDocuments({
  317. documentVersion: {
  318. $ne: requiredDocumentVersion[requiredDocumentVersion.length - 1]
  319. }
  320. });
  321. if (count2 > 0)
  322. console.warn(
  323. `Collection "${modelName}" has ${count2} documents with a outdated document version. Run steps manually to update these.`
  324. );
  325. }
  326. },
  327. err => {
  328. if (err) reject(new Error(err));
  329. else resolve();
  330. }
  331. );
  332. });
  333. }
  334. /**
  335. * Returns a database model
  336. *
  337. * @param {object} payload - object containing the payload
  338. * @param {object} payload.modelName - name of the model to get
  339. * @returns {Promise} - returns promise (reject, resolve)
  340. */
  341. GET_MODEL(payload) {
  342. return new Promise(resolve => {
  343. resolve(DBModule.models[payload.modelName]);
  344. });
  345. }
  346. /**
  347. * Returns a database schema
  348. *
  349. * @param {object} payload - object containing the payload
  350. * @param {object} payload.schemaName - name of the schema to get
  351. * @returns {Promise} - returns promise (reject, resolve)
  352. */
  353. GET_SCHEMA(payload) {
  354. return new Promise(resolve => {
  355. resolve(DBModule.schemas[payload.schemaName]);
  356. });
  357. }
  358. /**
  359. * Gets data
  360. *
  361. * @param {object} payload - object containing the payload
  362. * @param {string} payload.page - the page
  363. * @param {string} payload.pageSize - the page size
  364. * @param {string} payload.properties - the properties to return for each song
  365. * @param {string} payload.sort - the sort object
  366. * @param {string} payload.queries - the queries array
  367. * @param {string} payload.operator - the operator for queries
  368. * @param {string} payload.modelName - the db collection modal name
  369. * @param {string} payload.blacklistedProperties - the properties that are not allowed to be returned, filtered by or sorted by
  370. * @param {string} payload.specialProperties - the special properties
  371. * @param {string} payload.specialQueries - the special queries
  372. * @returns {Promise} - returns a promise (resolve, reject)
  373. */
  374. GET_DATA(payload) {
  375. return new Promise((resolve, reject) => {
  376. async.waterfall(
  377. [
  378. // Creates pipeline array
  379. next => next(null, []),
  380. // If a query filter property or sort property is blacklisted, throw error
  381. (pipeline, next) => {
  382. const { sort, queries, blacklistedProperties } = payload;
  383. if (
  384. queries.some(query =>
  385. blacklistedProperties.some(blacklistedProperty =>
  386. blacklistedProperty.startsWith(query.filter.property)
  387. )
  388. )
  389. )
  390. return next("Unable to filter by blacklisted property.");
  391. if (
  392. Object.keys(sort).some(property =>
  393. blacklistedProperties.some(blacklistedProperty =>
  394. blacklistedProperty.startsWith(property)
  395. )
  396. )
  397. )
  398. return next("Unable to sort by blacklisted property.");
  399. return next(null, pipeline);
  400. },
  401. // If a filter or property exists for a special property, add some custom pipeline steps
  402. (pipeline, next) => {
  403. const { properties, queries, specialProperties } = payload;
  404. async.eachLimit(
  405. Object.entries(specialProperties),
  406. 1,
  407. ([specialProperty, pipelineSteps], next) => {
  408. // Check if a filter with the special property exists
  409. const filterExists =
  410. queries.map(query => query.filter.property).indexOf(specialProperty) !== -1;
  411. // Check if a property with the special property exists
  412. const propertyExists = properties.indexOf(specialProperty) !== -1;
  413. // If no such filter or property exists, skip this function
  414. if (!filterExists && !propertyExists) return next();
  415. // Add the specified pipeline steps into the pipeline
  416. pipeline.push(...pipelineSteps);
  417. return next();
  418. },
  419. err => {
  420. next(err, pipeline);
  421. }
  422. );
  423. },
  424. // Adds the match stage to aggregation pipeline, which is responsible for filtering
  425. (pipeline, next) => {
  426. const { queries, operator, specialQueries, specialFilters } = payload;
  427. let queryError;
  428. const newQueries = queries.flatMap(query => {
  429. const { data, filter, filterType } = query;
  430. const newQuery = {};
  431. if (filterType === "regex") {
  432. newQuery[filter.property] = new RegExp(`${data.slice(1, data.length - 1)}`, "i");
  433. } else if (filterType === "contains") {
  434. newQuery[filter.property] = new RegExp(
  435. `${data.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&")}`,
  436. "i"
  437. );
  438. } else if (filterType === "exact") {
  439. newQuery[filter.property] = data.toString();
  440. } else if (filterType === "datetimeBefore") {
  441. newQuery[filter.property] = { $lte: new Date(data) };
  442. } else if (filterType === "datetimeAfter") {
  443. newQuery[filter.property] = { $gte: new Date(data) };
  444. } else if (filterType === "numberLesserEqual") {
  445. newQuery[filter.property] = { $lte: Number(data) };
  446. } else if (filterType === "numberLesser") {
  447. newQuery[filter.property] = { $lt: Number(data) };
  448. } else if (filterType === "numberGreater") {
  449. newQuery[filter.property] = { $gt: Number(data) };
  450. } else if (filterType === "numberGreaterEqual") {
  451. newQuery[filter.property] = { $gte: Number(data) };
  452. } else if (filterType === "numberEquals") {
  453. newQuery[filter.property] = { $eq: Number(data) };
  454. } else if (filterType === "boolean") {
  455. newQuery[filter.property] = { $eq: !!data };
  456. } else if (filterType === "special") {
  457. pipeline.push(...specialFilters[filter.property](data));
  458. newQuery[filter.property] = { $eq: true };
  459. }
  460. if (specialQueries[filter.property]) {
  461. return specialQueries[filter.property](newQuery);
  462. }
  463. return newQuery;
  464. });
  465. if (queryError) next(queryError);
  466. const queryObject = {};
  467. if (newQueries.length > 0) {
  468. if (operator === "and") queryObject.$and = newQueries;
  469. else if (operator === "or") queryObject.$or = newQueries;
  470. else if (operator === "nor") queryObject.$nor = newQueries;
  471. }
  472. pipeline.push({ $match: queryObject });
  473. next(null, pipeline);
  474. },
  475. // Adds sort stage to aggregation pipeline if there is at least one column being sorted, responsible for sorting data
  476. (pipeline, next) => {
  477. const { sort } = payload;
  478. const newSort = Object.fromEntries(
  479. Object.entries(sort).map(([property, direction]) => [
  480. property,
  481. direction === "ascending" ? 1 : -1
  482. ])
  483. );
  484. if (Object.keys(newSort).length > 0) pipeline.push({ $sort: newSort });
  485. next(null, pipeline);
  486. },
  487. // Adds first project stage to aggregation pipeline, responsible for including only the requested properties
  488. (pipeline, next) => {
  489. const { properties } = payload;
  490. pipeline.push({ $project: Object.fromEntries(properties.map(property => [property, 1])) });
  491. next(null, pipeline);
  492. },
  493. // Adds second project stage to aggregation pipeline, responsible for excluding some specific properties
  494. (pipeline, next) => {
  495. const { blacklistedProperties } = payload;
  496. if (blacklistedProperties.length > 0)
  497. pipeline.push({
  498. $project: Object.fromEntries(blacklistedProperties.map(property => [property, 0]))
  499. });
  500. next(null, pipeline);
  501. },
  502. // Adds the facet stage to aggregation pipeline, responsible for returning a total document count, skipping and limitting the documents that will be returned
  503. (pipeline, next) => {
  504. const { page, pageSize } = payload;
  505. pipeline.push({
  506. $facet: {
  507. count: [{ $count: "count" }],
  508. documents: [{ $skip: pageSize * (page - 1) }, { $limit: pageSize }]
  509. }
  510. });
  511. // console.dir(pipeline, { depth: 6 });
  512. next(null, pipeline);
  513. },
  514. (pipeline, next) => {
  515. const { modelName } = payload;
  516. DBModule.runJob("GET_MODEL", { modelName }, this)
  517. .then(model => {
  518. if (!model) return next("Invalid model.");
  519. return next(null, pipeline, model);
  520. })
  521. .catch(err => {
  522. next(err);
  523. });
  524. },
  525. // Executes the aggregation pipeline
  526. (pipeline, model, next) => {
  527. model.aggregate(pipeline).exec((err, result) => {
  528. // console.dir(err);
  529. // console.dir(result, { depth: 6 });
  530. if (err) return next(err);
  531. if (result[0].count.length === 0) return next(null, 0, []);
  532. const { count } = result[0].count[0];
  533. const { documents } = result[0];
  534. // console.log(111, err, result, count, documents[0]);
  535. return next(null, count, documents);
  536. });
  537. }
  538. ],
  539. (err, count, documents) => {
  540. if (err && err !== true) return reject(new Error(err));
  541. return resolve({ data: documents, count });
  542. }
  543. );
  544. });
  545. }
  546. /**
  547. * Checks if a password to be stored in the database has a valid length
  548. *
  549. * @param {object} password - the password itself
  550. * @returns {Promise} - returns promise (reject, resolve)
  551. */
  552. passwordValid(password) {
  553. return isLength(password, 6, 200);
  554. }
  555. }
  556. export default new _DBModule();