index.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435
  1. import "./loadEnvVariables.js";
  2. import util from "util";
  3. import config from "config";
  4. import fs from "fs";
  5. import package_json from "./package.json" assert { type: "json" };
  6. const REQUIRED_CONFIG_VERSION = 9;
  7. // eslint-disable-next-line
  8. Array.prototype.remove = function (item) {
  9. this.splice(this.indexOf(item), 1);
  10. };
  11. process.on("uncaughtException", err => {
  12. if (err.code === "ECONNREFUSED" || err.code === "UNCERTAIN_STATE") return;
  13. console.log(`UNCAUGHT EXCEPTION: ${err.stack}`);
  14. });
  15. const blacklistedConsoleLogs = [];
  16. const oldConsole = {};
  17. oldConsole.log = console.log;
  18. console.log = (...args) => {
  19. const string = util.format.apply(null, args);
  20. let blacklisted = false;
  21. blacklistedConsoleLogs.forEach(blacklistedConsoleLog => {
  22. if (string.indexOf(blacklistedConsoleLog) !== -1) blacklisted = true;
  23. });
  24. if (!blacklisted) oldConsole.log.apply(null, args);
  25. };
  26. const MUSARE_VERSION = package_json.version;
  27. const printVersion = () => {
  28. console.log(`Musare version: ${MUSARE_VERSION}.`);
  29. try {
  30. const head_contents = fs.readFileSync(".parent_git/HEAD").toString().replaceAll("\n", "");
  31. const branch = new RegExp("ref: refs/heads/([A-Za-z0-9_.-]+)").exec(head_contents)[1];
  32. const config_contents = fs.readFileSync(".parent_git/config").toString().replaceAll("\t", "").split("\n");
  33. const remote = new RegExp("remote = (.+)").exec(config_contents[config_contents.indexOf(`[branch "${branch}"]`) + 1])[1];
  34. const remote_url = new RegExp("url = (.+)").exec(config_contents[config_contents.indexOf(`[remote "${remote}"]`) + 1])[1];
  35. const latest_commit = fs.readFileSync(`.parent_git/refs/heads/${branch}`).toString().replaceAll("\n", "");
  36. const latest_commit_short = latest_commit.substr(0, 7);
  37. console.log(`Git branch: ${remote}/${branch}. Remote url: ${remote_url}. Latest commit: ${latest_commit} (${latest_commit_short}).`);
  38. } catch(e) {
  39. console.log(`Could not get Git info: ${e.message}.`);
  40. }
  41. }
  42. printVersion();
  43. if (
  44. (!config.has("configVersion") || config.get("configVersion") !== REQUIRED_CONFIG_VERSION) &&
  45. !(config.has("skipConfigVersionCheck") && config.get("skipConfigVersionCheck"))
  46. ) {
  47. console.log(
  48. "CONFIG VERSION IS WRONG. PLEASE UPDATE YOUR CONFIG WITH THE HELP OF THE TEMPLATE FILE AND THE README FILE."
  49. );
  50. process.exit();
  51. }
  52. if (config.debug && config.debug.traceUnhandledPromises === true) {
  53. console.log("Enabled trace-unhandled/register");
  54. import("trace-unhandled/register");
  55. }
  56. class JobManager {
  57. // eslint-disable-next-line require-jsdoc
  58. constructor() {
  59. this.runningJobs = {};
  60. }
  61. /**
  62. * Adds a job to the list of running jobs
  63. *
  64. * @param {object} job - the job object
  65. */
  66. addJob(job) {
  67. if (!this.runningJobs[job.module.name]) this.runningJobs[job.module.name] = {};
  68. this.runningJobs[job.module.name][job.toString()] = job;
  69. }
  70. /**
  71. * Removes a job from the list of running jobs (after it's completed)
  72. *
  73. * @param {object} job - the job object
  74. */
  75. removeJob(job) {
  76. if (!this.runningJobs[job.module.name]) this.runningJobs[job.module.name] = {};
  77. delete this.runningJobs[job.module.name][job.toString()];
  78. }
  79. /**
  80. * Returns detail about a job via a identifier
  81. *
  82. * @param {string} uuid - the job identifier
  83. * @returns {object} - the job object
  84. */
  85. getJob(uuid) {
  86. let job = null;
  87. Object.keys(this.runningJobs).forEach(moduleName => {
  88. if (this.runningJobs[moduleName][uuid]) job = this.runningJobs[moduleName][uuid];
  89. });
  90. return job;
  91. }
  92. }
  93. class ModuleManager {
  94. // eslint-disable-next-line require-jsdoc
  95. constructor() {
  96. this.modules = {};
  97. this.modulesNotInitialized = [];
  98. this.jobManager = new JobManager();
  99. this.i = 0;
  100. this.lockdown = false;
  101. this.debugLogs = {
  102. stationIssue: []
  103. };
  104. this.debugJobs = {
  105. all: [],
  106. completed: []
  107. };
  108. this.name = "MODULE_MANAGER";
  109. }
  110. /**
  111. * Adds a new module to the backend server/module manager
  112. *
  113. * @param {string} moduleName - the name of the module (also needs to be the same as the filename of a module located in the logic folder or "logic/moduleName/index.js")
  114. */
  115. async addModule(moduleName) {
  116. this.log("INFO", "Adding module", moduleName);
  117. this.modules[moduleName] = import(`./logic/${moduleName}`);
  118. }
  119. /**
  120. * Initialises a new module to the backend server/module manager
  121. *
  122. */
  123. async initialize() {
  124. this.reservedLines = Object.keys(this.modules).length + 5;
  125. await Promise.all(Object.values(this.modules)).then(modules => {
  126. for (let module = 0; module < modules.length; module += 1) {
  127. this.modules[modules[module].default.name] = modules[module].default;
  128. this.modulesNotInitialized.push(modules[module].default);
  129. }
  130. }); // ensures all modules are imported, then converts promise to the default export of the import
  131. Object.keys(this.modules).every(moduleKey => {
  132. const module = this.modules[moduleKey];
  133. module.setModuleManager(this);
  134. if (this.lockdown) return false;
  135. module._initialize();
  136. return true;
  137. });
  138. }
  139. /**
  140. * Called when a module is initialised
  141. *
  142. * @param {object} module - the module object/class
  143. */
  144. onInitialize(module) {
  145. if (this.modulesNotInitialized.indexOf(module) !== -1) {
  146. this.modulesNotInitialized.splice(this.modulesNotInitialized.indexOf(module), 1);
  147. this.log(
  148. "INFO",
  149. `Initialized: ${Object.keys(this.modules).length - this.modulesNotInitialized.length}/${
  150. Object.keys(this.modules).length
  151. }.`
  152. );
  153. if (this.modulesNotInitialized.length === 0) this.onAllModulesInitialized();
  154. }
  155. }
  156. /**
  157. * Called when a module fails to initialise
  158. *
  159. * @param {object} module - the module object/class
  160. */
  161. onFail(module) {
  162. if (this.modulesNotInitialized.indexOf(module) !== -1) {
  163. this.log("ERROR", "A module failed to initialize!");
  164. }
  165. }
  166. /**
  167. * Called when every module has initialised
  168. *
  169. */
  170. onAllModulesInitialized() {
  171. this.log("INFO", "All modules initialized!");
  172. }
  173. /**
  174. * Creates a new log message
  175. *
  176. * @param {...any} args - anything to be included in the log message, the first argument is the type of log
  177. */
  178. log(...args) {
  179. const _arguments = Array.from(args);
  180. const type = _arguments[0];
  181. _arguments.splice(0, 1);
  182. const start = `|${this.name.toUpperCase()}|`;
  183. const numberOfSpacesNeeded = 20 - start.length;
  184. _arguments.unshift(`${start}${Array(numberOfSpacesNeeded).join(" ")}`);
  185. if (type === "INFO") {
  186. _arguments[0] += "\x1b[36m";
  187. _arguments.push("\x1b[0m");
  188. console.log.apply(null, _arguments);
  189. } else if (type === "ERROR") {
  190. _arguments[0] += "\x1b[31m";
  191. _arguments.push("\x1b[0m");
  192. console.error.apply(null, _arguments);
  193. }
  194. }
  195. /**
  196. * Locks down all modules
  197. */
  198. _lockdown() {
  199. this.lockdown = true;
  200. Object.keys(this.modules).every(moduleKey => {
  201. const module = this.modules[moduleKey];
  202. module.setStatus("LOCKDOWN");
  203. return true;
  204. });
  205. }
  206. }
  207. const moduleManager = new ModuleManager();
  208. if (!config.get("migration")) {
  209. moduleManager.addModule("cache");
  210. moduleManager.addModule("db");
  211. moduleManager.addModule("mail");
  212. moduleManager.addModule("activities");
  213. moduleManager.addModule("api");
  214. moduleManager.addModule("app");
  215. moduleManager.addModule("ws");
  216. moduleManager.addModule("notifications");
  217. moduleManager.addModule("playlists");
  218. moduleManager.addModule("punishments");
  219. moduleManager.addModule("songs");
  220. moduleManager.addModule("stations");
  221. moduleManager.addModule("tasks");
  222. moduleManager.addModule("utils");
  223. moduleManager.addModule("youtube");
  224. } else {
  225. moduleManager.addModule("migration");
  226. }
  227. moduleManager.initialize();
  228. /**
  229. * Prints a job
  230. *
  231. * @param {object} job - the job
  232. * @param {number} layer - the layer
  233. */
  234. function printJob(job, layer) {
  235. const tabs = Array(layer).join("\t");
  236. if (job) {
  237. console.log(`${tabs}${job.name} (${job.toString()}) ${job.status}`);
  238. job.childJobs.forEach(childJob => {
  239. printJob(childJob, layer + 1);
  240. });
  241. } else console.log(`${tabs}JOB WAS REMOVED`);
  242. }
  243. /**
  244. * Prints a task
  245. *
  246. * @param {object} task - the task
  247. * @param {number} layer - the layer
  248. */
  249. function printTask(task, layer) {
  250. const tabs = Array(layer).join("\t");
  251. console.log(`${tabs}${task.job.name} (${task.job.toString()}) ${task.job.status} (priority: ${task.priority})`);
  252. task.job.childJobs.forEach(childJob => {
  253. printJob(childJob, layer + 1);
  254. });
  255. }
  256. import * as readline from 'node:readline';
  257. var rl = readline.createInterface({
  258. input: process.stdin,
  259. output: process.stdout,
  260. completer: function(command) {
  261. const parts = command.split(" ");
  262. const commands = ["version", "lockdown", "status", "running ", "queued ", "paused ", "stats ", "jobinfo ", "runjob ", "eval "];
  263. if (parts.length === 1) {
  264. const hits = commands.filter(c => c.startsWith(parts[0]));
  265. return [hits.length ? hits : commands, command];
  266. } else if (parts.length === 2) {
  267. if (["queued", "running", "paused", "runjob", "stats"].indexOf(parts[0]) !== -1) {
  268. const modules = Object.keys(moduleManager.modules);
  269. const hits = modules.filter(module => module.startsWith(parts[1])).map(module => `${parts[0]} ${module}${parts[0] === "runjob" ? " " : ""}`);
  270. return [hits.length ? hits : modules, command];
  271. } else {
  272. return [];
  273. }
  274. } else if (parts.length === 3) {
  275. if (parts[0] === "runjob") {
  276. const modules = Object.keys(moduleManager.modules);
  277. if (modules.indexOf(parts[1]) !== -1) {
  278. const jobs = moduleManager.modules[parts[1]].jobNames;
  279. const hits = jobs.filter(job => job.startsWith(parts[2])).map(job => `${parts[0]} ${parts[1]} ${job} `);
  280. return [hits.length ? hits : jobs, command];
  281. }
  282. } else {
  283. return [];
  284. }
  285. } else {
  286. return [];
  287. }
  288. }
  289. });
  290. rl.on("line",function(command) {
  291. if (command === "version") {
  292. printVersion();
  293. }
  294. if (command === "lockdown") {
  295. console.log("Locking down.");
  296. moduleManager._lockdown();
  297. }
  298. if (command === "status") {
  299. console.log("Status:");
  300. Object.keys(moduleManager.modules).forEach(moduleName => {
  301. const module = moduleManager.modules[moduleName];
  302. const tabsNeeded = 4 - Math.ceil((moduleName.length + 1) / 8);
  303. console.log(
  304. `${moduleName.toUpperCase()}${Array(tabsNeeded).join(
  305. "\t"
  306. )}${module.getStatus()}. Jobs in queue: ${module.jobQueue.lengthQueue()}. Jobs in progress: ${module.jobQueue.lengthRunning()}. Jobs paused: ${module.jobQueue.lengthPaused()} Concurrency: ${
  307. module.jobQueue.concurrency
  308. }. Stage: ${module.getStage()}`
  309. );
  310. });
  311. }
  312. if (command.startsWith("running")) {
  313. const parts = command.split(" ");
  314. moduleManager.modules[parts[1]].jobQueue.runningTasks.forEach(task => {
  315. printTask(task, 1);
  316. });
  317. }
  318. if (command.startsWith("queued")) {
  319. const parts = command.split(" ");
  320. moduleManager.modules[parts[1]].jobQueue.queue.forEach(task => {
  321. printTask(task, 1);
  322. });
  323. }
  324. if (command.startsWith("paused")) {
  325. const parts = command.split(" ");
  326. moduleManager.modules[parts[1]].jobQueue.pausedTasks.forEach(task => {
  327. printTask(task, 1);
  328. });
  329. }
  330. if (command.startsWith("stats")) {
  331. const parts = command.split(" ");
  332. console.log(moduleManager.modules[parts[1]].jobStatistics);
  333. }
  334. if (command.startsWith("jobinfo")) {
  335. const parts = command.split(" ");
  336. const uuid = parts[1];
  337. const jobFound = moduleManager.jobManager.getJob(uuid);
  338. if (jobFound) {
  339. let topParent = jobFound;
  340. let levelsDeep = 0;
  341. while (topParent.parentJob && topParent !== topParent.parentJob) {
  342. topParent = jobFound.parentJob;
  343. levelsDeep += 1;
  344. }
  345. console.log(
  346. `Found job, displaying that job and the full tree from the top parent job. The job is ${levelsDeep} levels deep from the top parent.`
  347. );
  348. console.log(jobFound);
  349. printJob(topParent, 1);
  350. } else console.log("Could not find job in job manager.");
  351. }
  352. if (command.startsWith("runjob")) {
  353. const parts = command.split(" ");
  354. const module = parts[1];
  355. const jobName = parts[2];
  356. const payload = JSON.parse(parts[3]);
  357. moduleManager.modules[module]
  358. .runJob(jobName, payload)
  359. .then(response => {
  360. console.log("runjob success", response);
  361. })
  362. .catch(err => {
  363. console.log("runjob error", err);
  364. });
  365. }
  366. if (command.startsWith("eval")) {
  367. const evalCommand = command.replace("eval ", "");
  368. console.log(`Running eval command: ${evalCommand}`);
  369. // eslint-disable-next-line no-eval
  370. const response = eval(evalCommand);
  371. console.log(`Eval response: `, response);
  372. }
  373. if (command.startsWith("debug")) {
  374. moduleManager.modules["youtube"].apiCalls.forEach(apiCall => {
  375. // console.log(`${apiCall.date.toISOString()} - ${apiCall.url} - ${apiCall.quotaCost} - ${JSON.stringify(apiCall.params)}`);
  376. console.log(apiCall);
  377. });
  378. }
  379. });
  380. export default moduleManager;