Ver Fonte

Added migration system, including migration 1

Kristian Vos há 4 anos atrás
pai
commit
cc1f6c2b52

+ 16 - 1
backend/config/template.json

@@ -1,5 +1,6 @@
 {
 	"mode": "development",
+    "migration": false,
 	"secret": "default",
 	"domain": "http://localhost",
 	"frontendPort": 80,
@@ -69,6 +70,20 @@
 
         //     ],
         //     "blacklistedTerms": []
-        // }
+		// },
+		"migration": {
+            "hideType": [
+
+            ],
+            "blacklistedTerms": [
+                "Ran job",
+                "Running job",
+                "Queuing job",
+                "Pausing job",
+                "is queued",
+                "is re-queued",
+                "Requeing"
+            ]
+        }
     }
 }

+ 19 - 15
backend/index.js

@@ -362,21 +362,25 @@ class ModuleManager {
 
 const moduleManager = new ModuleManager();
 
-moduleManager.addModule("cache");
-moduleManager.addModule("db");
-moduleManager.addModule("mail");
-moduleManager.addModule("activities");
-moduleManager.addModule("api");
-moduleManager.addModule("app");
-moduleManager.addModule("io");
-moduleManager.addModule("notifications");
-moduleManager.addModule("playlists");
-moduleManager.addModule("punishments");
-moduleManager.addModule("songs");
-moduleManager.addModule("stations");
-moduleManager.addModule("tasks");
-moduleManager.addModule("utils");
-moduleManager.addModule("youtube");
+if (!config.get("migration")) {
+	moduleManager.addModule("cache");
+	moduleManager.addModule("db");
+	moduleManager.addModule("mail");
+	moduleManager.addModule("activities");
+	moduleManager.addModule("api");
+	moduleManager.addModule("app");
+	moduleManager.addModule("io");
+	moduleManager.addModule("notifications");
+	moduleManager.addModule("playlists");
+	moduleManager.addModule("punishments");
+	moduleManager.addModule("songs");
+	moduleManager.addModule("stations");
+	moduleManager.addModule("tasks");
+	moduleManager.addModule("utils");
+	moduleManager.addModule("youtube");
+} else {
+	moduleManager.addModule("migration");
+}
 
 moduleManager.initialize();
 

+ 2 - 1
backend/logic/db/schemas/activity.js

@@ -16,5 +16,6 @@ export default {
 		],
 		required: true
 	},
-	payload: { type: Array, required: true }
+	payload: { type: Array, required: true },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/news.js

@@ -6,5 +6,6 @@ export default {
 	improvements: [{ type: String }],
 	upcoming: [{ type: String }],
 	createdBy: { type: String, required: true },
-	createdAt: { type: Number, default: Date.now, required: true }
+	createdAt: { type: Number, default: Date.now, required: true },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/playlist.js

@@ -13,5 +13,6 @@ export default {
 	createdAt: { type: Date, default: Date.now, required: true },
 	createdFor: { type: String },
 	privacy: { type: String, enum: ["public", "private"], default: "private" },
-	type: { type: String, enum: ["user", "genre"], required: true }
+	type: { type: String, enum: ["user", "genre"], required: true },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/punishment.js

@@ -5,5 +5,6 @@ export default {
 	active: { type: Boolean, required: true, default: true },
 	expiresAt: { type: Date, required: true },
 	punishedAt: { type: Date, default: Date.now, required: true },
-	punishedBy: { type: String, required: true }
+	punishedBy: { type: String, required: true },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/queueSong.js

@@ -9,5 +9,6 @@ export default {
 	explicit: { type: Boolean, required: true },
 	requestedBy: { type: String, required: true },
 	requestedAt: { type: Date, default: Date.now, required: true },
-	discogs: { type: Object }
+	discogs: { type: Object },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/report.js

@@ -12,5 +12,6 @@ export default {
 		}
 	],
 	createdBy: { type: String, required: true },
-	createdAt: { type: Date, default: Date.now, required: true }
+	createdAt: { type: Date, default: Date.now, required: true },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/song.js

@@ -13,5 +13,6 @@ export default {
 	requestedAt: { type: Date, required: true },
 	acceptedBy: { type: String, required: true },
 	acceptedAt: { type: Date, default: Date.now, required: true },
-	discogs: { type: Object }
+	discogs: { type: Object },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/station.js

@@ -45,5 +45,6 @@ export default {
 	owner: { type: String },
 	privatePlaylist: { type: mongoose.Schema.Types.ObjectId },
 	partyMode: { type: Boolean },
-	theme: { type: String, enum: ["blue", "purple", "teal", "orange"], default: "blue" }
+	theme: { type: String, enum: ["blue", "purple", "teal", "orange"], default: "blue" },
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 2 - 1
backend/logic/db/schemas/user.js

@@ -43,5 +43,6 @@ export default {
 		orderOfPlaylists: [{ type: mongoose.Schema.Types.ObjectId }],
 		nightmode: { type: Boolean, default: false, required: true },
 		autoSkipDisliked: { type: Boolean, default: true, required: true }
-	}
+	},
+	documentVersion: { type: Number, default: 1, required: true }
 };

+ 144 - 0
backend/logic/migration/index.js

@@ -0,0 +1,144 @@
+import async from "async";
+import config from "config";
+import mongoose from "mongoose";
+import bluebird from "bluebird";
+import fs from "fs";
+
+import { fileURLToPath } from "url";
+import path from "path";
+
+import CoreClass from "../../core";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+let MigrationModule;
+
+mongoose.Promise = bluebird;
+
+class _MigrationModule extends CoreClass {
+	// eslint-disable-next-line require-jsdoc
+	constructor() {
+		super("migration");
+
+		MigrationModule = this;
+	}
+
+	/**
+	 * Initialises the migration module
+	 *
+	 * @returns {Promise} - returns promise (reject, resolve)
+	 */
+	initialize() {
+		return new Promise((resolve, reject) => {
+			this.models = {};
+
+			const mongoUrl = config.get("mongo").url;
+
+			mongoose
+				.connect(mongoUrl, {
+					useNewUrlParser: true,
+					useUnifiedTopology: true,
+					useCreateIndex: true
+				})
+				.then(async () => {
+					mongoose.connection.on("error", err => {
+						this.log("ERROR", err);
+					});
+
+					mongoose.connection.on("disconnected", () => {
+						this.log("ERROR", "Disconnected, going to try to reconnect...");
+						this.setStatus("RECONNECTING");
+					});
+
+					mongoose.connection.on("reconnected", () => {
+						this.log("INFO", "Reconnected.");
+						this.setStatus("READY");
+					});
+
+					mongoose.connection.on("reconnectFailed", () => {
+						this.log("INFO", "Reconnect failed, stopping reconnecting.");
+						// this.failed = true;
+						// this._lockdown();
+						this.setStatus("FAILED");
+					});
+
+					this.models = {
+						song: mongoose.model("song", new mongoose.Schema({}, { strict: false })),
+						queueSong: mongoose.model("queueSong", new mongoose.Schema({}, { strict: false })),
+						station: mongoose.model("station", new mongoose.Schema({}, { strict: false })),
+						user: mongoose.model("user", new mongoose.Schema({}, { strict: false })),
+						activity: mongoose.model("activity", new mongoose.Schema({}, { strict: false })),
+						playlist: mongoose.model("playlist", new mongoose.Schema({}, { strict: false })),
+						news: mongoose.model("news", new mongoose.Schema({}, { strict: false })),
+						report: mongoose.model("report", new mongoose.Schema({}, { strict: false })),
+						punishment: mongoose.model("punishment", new mongoose.Schema({}, { strict: false }))
+					};
+
+					const files = fs.readdirSync(path.join(__dirname, "migrations"));
+					const migrations = files.length;
+
+					async.timesLimit(
+						migrations,
+						1,
+						(index, next) => {
+							MigrationModule.runJob("RUN_MIGRATION", { index: index + 1 }, null, -1)
+								.then(() => {
+									next();
+								})
+								.catch(err => {
+									next(err);
+								});
+						},
+						err => {
+							if (err) console.log("Migration error", err);
+						}
+					);
+
+					resolve();
+				})
+				.catch(err => {
+					this.log("ERROR", err);
+					reject(err);
+				});
+		});
+	}
+
+	/**
+	 * Returns a database model
+	 *
+	 * @param {object} payload - object containing the payload
+	 * @param {object} payload.modelName - name of the model to get
+	 * @returns {Promise} - returns promise (reject, resolve)
+	 */
+	GET_MODEL(payload) {
+		return new Promise(resolve => {
+			resolve(MigrationModule.models[payload.modelName]);
+		});
+	}
+
+	/**
+	 * Runs migrations
+	 *
+	 * @param {object} payload - object containing the payload
+	 * @param {object} payload.index - migration index
+	 * @returns {Promise} - returns promise (reject, resolve)
+	 */
+	RUN_MIGRATION(payload) {
+		return new Promise((resolve, reject) => {
+			import(`./migrations/migration${payload.index}`).then(module => {
+				this.log("INFO", `Running migration ${payload.index}`);
+				module.default
+					.apply(this, [MigrationModule])
+					.then(response => {
+						resolve(response);
+					})
+					.catch(err => {
+						reject(err);
+					});
+			});
+		});
+	}
+}
+
+export default new _MigrationModule();

+ 181 - 0
backend/logic/migration/migrations/migration1.js

@@ -0,0 +1,181 @@
+import async from "async";
+
+/**
+ * Migration 1
+ *
+ * This migration is used to set the documentVersion to 1 for all documents that don't have a documentVersion yet, meaning they were created before the migration system
+ *
+ * @param {object} MigrationModule - the MigrationModule
+ * @returns {Promise} - returns promise
+ */
+export default async function migrate(MigrationModule) {
+	const activityModel = await MigrationModule.runJob("GET_MODEL", { modelName: "activity" }, this);
+	const newsModel = await MigrationModule.runJob("GET_MODEL", { modelName: "news" }, this);
+	const playlistModel = await MigrationModule.runJob("GET_MODEL", { modelName: "playlist" }, this);
+	const punishmentModel = await MigrationModule.runJob("GET_MODEL", { modelName: "punishment" }, this);
+	const queueSongModel = await MigrationModule.runJob("GET_MODEL", { modelName: "queueSong" }, this);
+	const reportModel = await MigrationModule.runJob("GET_MODEL", { modelName: "report" }, this);
+	const songModel = await MigrationModule.runJob("GET_MODEL", { modelName: "song" }, this);
+	const stationModel = await MigrationModule.runJob("GET_MODEL", { modelName: "station" }, this);
+	const userModel = await MigrationModule.runJob("GET_MODEL", { modelName: "user" }, this);
+
+	return new Promise((resolve, reject) => {
+		async.waterfall(
+			[
+				next => {
+					activityModel.updateMany(
+						{ documentVersion: null },
+						{ $set: { documentVersion: 1 } },
+						(err, res) => {
+							if (err) next(err);
+							else {
+								// { n: X, nModified: 1, ok: 1 }
+								// n = matched
+								// nModified = modified
+								// ok = successful
+
+								this.log(
+									"INFO",
+									`Migration 1 (activity). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+								);
+
+								next();
+							}
+						}
+					);
+				},
+
+				next => {
+					newsModel.updateMany({ documentVersion: null }, { $set: { documentVersion: 1 } }, (err, res) => {
+						if (err) next(err);
+						else {
+							this.log(
+								"INFO",
+								`Migration 1 (news). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+							);
+
+							next();
+						}
+					});
+				},
+
+				next => {
+					playlistModel.updateMany(
+						{ documentVersion: null },
+						{ $set: { documentVersion: 1 } },
+						(err, res) => {
+							if (err) next(err);
+							else {
+								this.log(
+									"INFO",
+									`Migration 1 (playlist). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+								);
+
+								next();
+							}
+						}
+					);
+				},
+
+				next => {
+					punishmentModel.updateMany(
+						{ documentVersion: null },
+						{ $set: { documentVersion: 1 } },
+						(err, res) => {
+							if (err) next(err);
+							else {
+								this.log(
+									"INFO",
+									`Migration 1 (punishment). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+								);
+
+								next();
+							}
+						}
+					);
+				},
+
+				next => {
+					queueSongModel.updateMany(
+						{ documentVersion: null },
+						{ $set: { documentVersion: 1 } },
+						(err, res) => {
+							if (err) next(err);
+							else {
+								this.log(
+									"INFO",
+									`Migration 1 (queueSong). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+								);
+
+								next();
+							}
+						}
+					);
+				},
+
+				next => {
+					reportModel.updateMany({ documentVersion: null }, { $set: { documentVersion: 1 } }, (err, res) => {
+						if (err) next(err);
+						else {
+							this.log(
+								"INFO",
+								`Migration 1 (report). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+							);
+
+							next();
+						}
+					});
+				},
+
+				next => {
+					songModel.updateMany({ documentVersion: null }, { $set: { documentVersion: 1 } }, (err, res) => {
+						if (err) next(err);
+						else {
+							this.log(
+								"INFO",
+								`Migration 1 (song). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+							);
+
+							next();
+						}
+					});
+				},
+
+				next => {
+					stationModel.updateMany({ documentVersion: null }, { $set: { documentVersion: 1 } }, (err, res) => {
+						if (err) next(err);
+						else {
+							this.log(
+								"INFO",
+								`Migration 1 (station). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+							);
+
+							next();
+						}
+					});
+				},
+
+				next => {
+					userModel.updateMany({ documentVersion: null }, { $set: { documentVersion: 1 } }, (err, res) => {
+						if (err) next(err);
+						else {
+							this.log(
+								"INFO",
+								`Migration 1 (user). Matched: ${res.n}, modified: ${res.nModified}, ok: ${res.ok}.`
+							);
+
+							next();
+						}
+					});
+				}
+			],
+			(err, response) => {
+				if (err) {
+					reject(new Error(err));
+				} else {
+					resolve(response);
+				}
+			}
+		);
+	});
+}