refactor: added code to handle disconnecting for redis, mongo and Discord with module states

backend/core.js

@@ -86,6 +86,7 @@ module.exports = class {
 	}
 
 	_lockdown() {
+		if (this.lockdown) return;
 		this.lockdown = true;
 		this.setState("LOCKDOWN");
 		this.moduleManager.printStatus();

backend/index.js

@@ -94,10 +94,10 @@ class ModuleManager {
 
 			let stateColor;
 			if (module.state === "NOT_INITIALIZED") stateColor = colors.FgWhite;
-			if (module.state === "INITIALIZING") stateColor = colors.FgYellow;
-			if (module.state === "INITIALIZED") stateColor = colors.FgGreen;
-			if (module.state === "LOCKDOWN" && !module.failed) stateColor = colors.FgRed;
-			if (module.state === "LOCKDOWN" && module.failed) stateColor = colors.FgMagenta;
+			else if (module.state === "INITIALIZED") stateColor = colors.FgGreen;
+			else if (module.state === "LOCKDOWN" && !module.failed) stateColor = colors.FgRed;
+			else if (module.state === "LOCKDOWN" && module.failed) stateColor = colors.FgMagenta;
+			else stateColor = colors.FgYellow;
 			
 			process.stdout.write(`${moduleName}${tabs}${stateColor}${module.state}\t${colors.FgYellow}Stage: ${colors.FgRed}${module.stage}${colors.FgWhite}. ${colors.FgYellow}Timing${colors.FgWhite}: [${timing}]${colors.FgWhite}${colors.FgWhite}. ${colors.FgYellow}Total time${colors.FgWhite}: ${colors.FgRed}${module.totalTimeInitialize}${colors.FgCyan}ms${colors.Reset}\n`);
 		}

backend/logic/cache/index.js

@@ -31,18 +31,38 @@ module.exports = class extends coreClass {
 
 			this.client = redis.createClient({
 				url: this.url,
-				password: this.password
+				password: this.password,
+				retry_strategy: (options) => {
+					if (this.state === "LOCKDOWN") return;
+					if (this.state !== "RECONNECTING") this.setState("RECONNECTING");
+
+					this.logger.info("CACHE_MODULE", `Attempting to reconnect.`);
+
+					if (options.attempt >= 10) {
+						this.logger.error("CACHE_MODULE", `Stopped trying to reconnect.`);
+
+						this.failed = true;
+						this._lockdown();
+
+						return undefined;
+					}
+
+					return 3000;
+				}
 			});
 
 			this.client.on('error', err => {
-				if (this.lockdown) return;
-				//errorCb('Cache connection error.', err, 'Cache');
-				console.log("REDIS ERROR " + err);
-				reject(err);
+				if (this.state === "INITIALIZING") reject(err);
+				if(this.state === "LOCKDOWN") return;
+
+				this.logger.error("CACHE_MODULE", `Error ${err.message}.`);
 			});
 
 			this.client.on("connect", () => {
-				resolve();
+				this.logger.info("CACHE_MODULE", "Connected succesfully.");
+
+				if (this.state === "INITIALIZING") resolve();
+				else if (this.state === "LOCKDOWN" || this.state === "RECONNECTING") this.setState("INITIALIZED");
 			});
 		});
 	}

backend/logic/db/index.js

@@ -33,7 +33,9 @@ module.exports = class extends coreClass {
 
 			mongoose.connect(mongoUrl, {
 				useNewUrlParser: true,
-				useCreateIndex: true
+				useCreateIndex: true,
+				reconnectInterval: 3000,
+				reconnectTries: 10
 			})
 				.then(() => {
 					this.schemas = {
@@ -57,6 +59,26 @@ module.exports = class extends coreClass {
 						report: mongoose.model('report', this.schemas.report),
 						punishment: mongoose.model('punishment', this.schemas.punishment)
 					};
+
+					mongoose.connection.on('error', err => {
+						this.logger.error("DB_MODULE", err);
+					});
+
+					mongoose.connection.on('disconnected', () => {
+						this.logger.error("DB_MODULE", "Disconnected, going to try to reconnect...");
+						this.setState("RECONNECTING");
+					});
+
+					mongoose.connection.on('reconnected', () => {
+						this.logger.success("DB_MODULE", "Reconnected.");
+						this.setState("INITIALIZED");
+					});
+
+					mongoose.connection.on('reconnectFailed', () => {
+						this.logger.error("DB_MODULE", "Reconnect failed, stopping reconnecting.");
+						this.failed = true;
+						this._lockdown();
+					});
 		
 					// this.schemas.user.path('username').validate((username) => {
 					// 	return (isLength(username, 2, 32) && regex.azAZ09_.test(username));
@@ -194,7 +216,7 @@ module.exports = class extends coreClass {
 					resolve();
 				})
 				.catch(err => {
-					console.error(err);
+					this.logger.error("DB_MODULE", err);
 					reject(err);
 				});
 		})

backend/logic/discord.js

@@ -12,38 +12,35 @@ module.exports = class extends coreClass {
 			this.setStage(1);
 
 			this.client = new Discord.Client();
-			
-			this.connected = false;
 			this.adminAlertChannelId = config.get("apis.discord").loggingChannel;
 			
 			this.client.on("ready", () => {
-				this.logger.info("DISCORD_READY", `Logged in as ${this.client.user.tag}!`);
-				this.connected = true;
-
-				//bus.emit("discordConnected");
+				this.logger.info("DISCORD_MODULE", `Logged in as ${this.client.user.tag}!`);
 
-				resolve();
-
-				/*messagesToSend.forEach(message => {
-					this.sendAdminAlertMessage(message.message, message.color, message.type, message.critical, message.extraFields);
-				});
-				messagesToSend = [];*/
+				if (this.state === "INITIALIZING") resolve();
+				else {
+					this.logger.info("DISCORD_MODULE", `Discord client reconnected.`);
+					this.setState("INITIALIZED");
+				}
 			});
 		  
 			this.client.on("disconnect", () => {
-				this.logger.info("DISCORD_DISCONNECT", `Discord client was disconnected.`);
-				this.connected = false;
+				this.logger.info("DISCORD_MODULE", `Discord client disconnected.`);
+
+				if (this.state === "INITIALIZING") reject();
+				else {
+					this.failed = true;
+					this._lockdown;
+				} 
 			});
 
 			this.client.on("reconnecting", () => {
-				this.logger.info("DISCORD_RECONNECTING", `Discord client reconnecting.`);
-				this.connected = false;
+				this.logger.info("DISCORD_MODULE", `Discord client reconnecting.`);
+				this.setState("RECONNECTING");
 			});
 		
 			this.client.on("error", err => {
-				this.logger.info("DISCORD_ERROR", `Discord client encountered an error: ${err.message}.`);
-
-				reject();
+				this.logger.info("DISCORD_MODULE", `Discord client encountered an error: ${err.message}.`);
 			});
 
 			this.client.login(config.get("apis.discord").token);
@@ -51,7 +48,7 @@ module.exports = class extends coreClass {
 	}
 
 	async sendAdminAlertMessage(message, color, type, critical, extraFields) {
-		try { await this._validateHook(); await this.connectedHook(); } catch { return; }
+		try { await this._validateHook(); } catch { return; }
 
 		const channel = this.client.channels.find("id", this.adminAlertChannelId);
 		if (channel !== null) {
@@ -91,13 +88,4 @@ module.exports = class extends coreClass {
 			this.logger.error("SEND_ADMIN_ALERT_MESSAGE", "Couldn't send admin alert message, channel was not found.");
 		}
 	}
-
-	connectedHook() {
-		return Promise.race([
-			new Promise(resolve => bus.once("discordConnected", resolve)),
-			new Promise(resolve => {
-				if (this.connected) resolve();
-			})
-		]);
-	}
 }

backend/logic/notifications.js

@@ -16,16 +16,76 @@ module.exports = class extends coreClass {
 			const url = this.url = config.get("redis").url;
 			const password = this.password = config.get("redis").password;
 
-			this.pub = redis.createClient({ url, password });
-			this.sub = redis.createClient({ url, password });
+			this.pub = redis.createClient({
+				url,
+				password,
+				retry_strategy: (options) => {
+					if (this.state === "LOCKDOWN") return;
+					if (this.state !== "RECONNECTING") this.setState("RECONNECTING");
+
+					this.logger.info("NOTIFICATIONS_MODULE", `Attempting to reconnect pub.`);
+
+					if (options.attempt >= 10) {
+						this.logger.error("NOTIFICATIONS_MODULE", `Stopped trying to reconnect pub.`);
+
+						this.failed = true;
+						this._lockdown();
+
+						return undefined;
+					}
+
+					return 3000;
+				}
+			});
+			this.sub = redis.createClient({
+				url,
+				password,
+				retry_strategy: (options) => {
+					if (this.state === "LOCKDOWN") return;
+					if (this.state !== "RECONNECTING") this.setState("RECONNECTING");
+
+					this.logger.info("NOTIFICATIONS_MODULE", `Attempting to reconnect sub.`);
+
+					if (options.attempt >= 10) {
+						this.logger.error("NOTIFICATIONS_MODULE", `Stopped trying to reconnect sub.`);
+
+						this.failed = true;
+						this._lockdown();
+
+						return undefined;
+					}
+
+					return 3000;
+				}
+			});
 
 			this.sub.on('error', (err) => {
-				errorCb('Cache connection error.', err, 'Notifications');
-				reject(err);
+				if (this.state === "INITIALIZING") reject(err);
+				if(this.state === "LOCKDOWN") return;
+
+				this.logger.error("NOTIFICATIONS_MODULE", `Sub error ${err.message}.`);
+			});
+
+			this.pub.on('error', (err) => {
+				if (this.state === "INITIALIZING") reject(err);
+				if(this.state === "LOCKDOWN") return; 
+
+				this.logger.error("NOTIFICATIONS_MODULE", `Pub error ${err.message}.`);
 			});
 
 			this.sub.on("connect", () => {
-				resolve();
+				this.logger.info("NOTIFICATIONS_MODULE", "Sub connected succesfully.");
+
+				if (this.state === "INITIALIZING") resolve();
+				else if (this.state === "LOCKDOWN" || this.state === "RECONNECTING") this.setState("INITIALIZED");
+				
+			});
+
+			this.pub.on("connect", () => {
+				this.logger.info("NOTIFICATIONS_MODULE", "Pub connected succesfully.");
+
+				if (this.state === "INITIALIZING") resolve();
+				else if (this.state === "LOCKDOWN" || this.state === "RECONNECTING") this.setState("INITIALIZED");
 			});
 
 			this.sub.on('pmessage', (pattern, channel, expiredKey) => {