core.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661
  1. import config from "config";
  2. class DeferredPromise {
  3. // eslint-disable-next-line require-jsdoc
  4. constructor() {
  5. this.promise = new Promise((resolve, reject) => {
  6. this.reject = reject;
  7. this.resolve = resolve;
  8. });
  9. }
  10. }
  11. class QueueTask {
  12. // eslint-disable-next-line require-jsdoc
  13. constructor(job, options, priority) {
  14. this.job = job;
  15. this.options = options;
  16. this.priority = priority;
  17. this.job.setTask(this);
  18. }
  19. }
  20. class Queue {
  21. // eslint-disable-next-line require-jsdoc
  22. constructor(handleTaskFunction, concurrency) {
  23. this.handleTaskFunction = handleTaskFunction;
  24. this.concurrency = concurrency;
  25. this.queue = [];
  26. this.runningTasks = [];
  27. this.pausedTasks = [];
  28. this.paused = false;
  29. }
  30. /**
  31. * Pauses the queue, meaning no new jobs can be started. Jobs can still be added to the queue, and already running tasks won't be paused.
  32. */
  33. pause() {
  34. this.paused = true;
  35. }
  36. /**
  37. * Resumes the queue.
  38. */
  39. resume() {
  40. this.paused = false;
  41. setTimeout(() => {
  42. this._handleQueue();
  43. }, 0);
  44. }
  45. /**
  46. * Returns the amount of jobs in the queue.
  47. *
  48. * @returns {number} - amount of jobs in queue
  49. */
  50. lengthQueue() {
  51. return this.queue.length;
  52. }
  53. /**
  54. * Returns the amount of running jobs.
  55. *
  56. * @returns {number} - amount of running jobs
  57. */
  58. lengthRunning() {
  59. return this.runningTasks.length;
  60. }
  61. /**
  62. * Returns the amount of running jobs.
  63. *
  64. * @returns {number} - amount of running jobs
  65. */
  66. lengthPaused() {
  67. return this.pausedTasks.length;
  68. }
  69. /**
  70. * Adds a job to the queue, with a given priority.
  71. *
  72. * @param {object} job - the job that is to be added
  73. * @param {object} options - custom options e.g. isQuiet. Optional.
  74. * @param {number} priority - the priority of the to be added job
  75. */
  76. push(job, options, priority) {
  77. this.queue.push(new QueueTask(job, options, priority));
  78. setTimeout(() => {
  79. this._handleQueue();
  80. }, 0);
  81. }
  82. /**
  83. * Removes a job currently running from the queue.
  84. *
  85. * @param {object} job - the job to be removed
  86. */
  87. removeRunningJob(job) {
  88. this.runningTasks.remove(this.runningTasks.find(task => task.job.toString() === job.toString()));
  89. }
  90. /**
  91. * Pauses a job currently running from the queue.
  92. *
  93. * @param {object} job - the job to be pauses
  94. */
  95. pauseRunningJob(job) {
  96. const task = this.runningTasks.find(task => task.job.toString() === job.toString());
  97. if (!task) {
  98. console.log(
  99. `Attempted to pause job ${job.name} (${job.toString()}), but couldn't find it in running tasks.`
  100. );
  101. return;
  102. }
  103. this.runningTasks.remove(task);
  104. this.pausedTasks.push(task);
  105. }
  106. /**
  107. * Resumes a job currently paused, adding the job back to the front of the queue
  108. *
  109. * @param {object} job - the job to be pauses
  110. */
  111. resumeRunningJob(job) {
  112. const task = this.pausedTasks.find(task => task.job.toString() === job.toString());
  113. if (!task) {
  114. console.log(
  115. `Attempted to resume job ${job.name} (${job.toString()}), but couldn't find it in paused tasks.`
  116. );
  117. return;
  118. }
  119. this.pausedTasks.remove(task);
  120. this.queue.unshift(task);
  121. setTimeout(() => {
  122. this._handleQueue();
  123. }, 0);
  124. }
  125. /**
  126. * Check if there's room for a job to be processed, and if there is, run it.
  127. */
  128. _handleQueue() {
  129. if (this.queue.length > 0) {
  130. const task = this.queue.reduce((a, b) => (a.priority < b.priority ? a : b));
  131. // console.log(`First task: `, task);
  132. if (task) {
  133. if ((!this.paused && this.runningTasks.length < this.concurrency) || task.priority === -1) {
  134. this.queue.remove(task);
  135. this.runningTasks.push(task);
  136. this._handleTask(task);
  137. setTimeout(() => {
  138. this._handleQueue();
  139. }, 0);
  140. }
  141. }
  142. }
  143. }
  144. /**
  145. * Handles a task, calling the handleTaskFunction provided in the constructor
  146. *
  147. * @param {object} task - the task to be handled
  148. */
  149. _handleTask(task) {
  150. this.handleTaskFunction(task.job, task.options).finally(() => {
  151. this.runningTasks.remove(task);
  152. this._handleQueue();
  153. });
  154. }
  155. }
  156. class Job {
  157. // eslint-disable-next-line require-jsdoc
  158. constructor(name, payload, onFinish, module, parentJob) {
  159. this.name = name;
  160. this.payload = payload;
  161. this.response = null;
  162. this.responseType = null;
  163. this.onFinish = onFinish;
  164. this.module = module;
  165. this.parentJob = parentJob;
  166. this.childJobs = [];
  167. /* eslint-disable no-bitwise, eqeqeq */
  168. this.uniqueId = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, c => {
  169. const r = (Math.random() * 16) | 0;
  170. const v = c == "x" ? r : (r & 0x3) | 0x8;
  171. return v.toString(16);
  172. });
  173. this.status = "INITIALIZED";
  174. this.task = null;
  175. }
  176. /**
  177. * Adds a child job to this job
  178. *
  179. * @param {object} childJob - the child job
  180. */
  181. addChildJob(childJob) {
  182. this.childJobs.push(childJob);
  183. }
  184. /**
  185. * Sets the job status
  186. *
  187. * @param {string} status - the new status
  188. */
  189. setStatus(status) {
  190. // console.log(`Job ${this.toString()} has changed status from ${this.status} to ${status}`);
  191. this.status = status;
  192. }
  193. /**
  194. * Sets the task for a job
  195. *
  196. * @param {string} task - the job task
  197. */
  198. setTask(task) {
  199. this.task = task;
  200. }
  201. /**
  202. * Returns the UUID of the job, allowing you to compare jobs with toString
  203. *
  204. * @returns {string} - the job's UUID/uniqueId
  205. */
  206. toString() {
  207. return this.uniqueId;
  208. }
  209. /**
  210. * Sets the response that will be provided to the onFinish DeferredPromise resolve/reject function, as soon as the job is done if it has no parent, or when the parent job is resumed
  211. *
  212. * @param {object} response - the response
  213. */
  214. setResponse(response) {
  215. this.response = response;
  216. }
  217. /**
  218. * Sets the response type that is paired with the response. If it is RESOLVE/REJECT, then it will resolve/reject with the response. If it is RESOLVED/REJECTED, then it has already resolved/rejected with the response.
  219. *
  220. * @param {string} responseType - the response type, so RESOLVE/REJECT/RESOLVED/REJECTED
  221. */
  222. setResponseType(responseType) {
  223. this.responseType = responseType;
  224. }
  225. /**
  226. * Removes child jobs to prevent memory leak
  227. */
  228. cleanup() {
  229. this.childJobs = this.childJobs.map(() => null);
  230. }
  231. /**
  232. * Logs to the module of the job
  233. *
  234. * @param {any} args - Anything to be added to the log e.g. log type, log message
  235. */
  236. log(...args) {
  237. args.splice(1, 0, this.name); // Adds the name of the job as the first argument (after INFO/SUCCESS/ERROR).
  238. this.module.log(...args);
  239. }
  240. }
  241. class MovingAverageCalculator {
  242. // eslint-disable-next-line require-jsdoc
  243. constructor() {
  244. this.count = 0;
  245. this._mean = 0;
  246. }
  247. /**
  248. * Updates the mean average
  249. *
  250. * @param {number} newValue - the new time it took to complete a job
  251. */
  252. update(newValue) {
  253. this.count += 1;
  254. const differential = (newValue - this._mean) / this.count;
  255. this._mean += differential;
  256. }
  257. /**
  258. * Returns the mean average
  259. *
  260. * @returns {number} - returns the mean average
  261. */
  262. get mean() {
  263. this.validate();
  264. return this._mean;
  265. }
  266. /**
  267. * Checks that the mean is valid
  268. */
  269. validate() {
  270. if (this.count === 0) throw new Error("Mean is undefined");
  271. }
  272. }
  273. export default class CoreClass {
  274. /**
  275. *
  276. * @param {string} name - the name of the class
  277. * @param {object} options - optional options
  278. * @param {number} options.concurrency - how many jobs can run at the same time
  279. * @param {object} options.priorities - custom priorities for jobs
  280. */
  281. constructor(name, options) {
  282. this.name = name;
  283. this.status = "UNINITIALIZED";
  284. // this.log("Core constructor");
  285. this.concurrency = options && options.concurrency ? options.concurrency : 10;
  286. this.jobQueue = new Queue((job, options) => this._runJob(job, options), this.concurrency);
  287. this.jobQueue.pause();
  288. this.priorities = options && options.priorities ? options.priorities : {};
  289. this.stage = 0;
  290. this.jobStatistics = {};
  291. this.logRules = config.get("customLoggingPerModule")[name]
  292. ? config.get("customLoggingPerModule")[name]
  293. : config.get("defaultLogging");
  294. this.registerJobs();
  295. }
  296. /**
  297. * Sets the status of a module
  298. *
  299. * @param {string} status - the new status of a module
  300. */
  301. setStatus(status) {
  302. this.status = status;
  303. this.log("INFO", `Status changed to: ${status}`);
  304. if (this.status === "READY") this.jobQueue.resume();
  305. else if (this.status === "FAIL" || this.status === "LOCKDOWN") this.jobQueue.pause();
  306. }
  307. /**
  308. * Returns the status of a module
  309. *
  310. * @returns {string} - the status of a module
  311. */
  312. getStatus() {
  313. return this.status;
  314. }
  315. /**
  316. * Changes the current stage of a module
  317. *
  318. * @param {string} stage - the new stage of a module
  319. */
  320. setStage(stage) {
  321. this.stage = stage;
  322. }
  323. /**
  324. * Returns the current stage of a module
  325. *
  326. * @returns {string} - the current stage of a module
  327. */
  328. getStage() {
  329. return this.stage;
  330. }
  331. /**
  332. * Initialises a module and handles initialise successes and failures
  333. */
  334. _initialize() {
  335. this.setStatus("INITIALIZING");
  336. this.initialize()
  337. .then(() => {
  338. this.setStatus("READY");
  339. this.moduleManager.onInitialize(this);
  340. })
  341. .catch(err => {
  342. console.error(err);
  343. this.setStatus("FAILED");
  344. this.moduleManager.onFail(this);
  345. });
  346. }
  347. /**
  348. * Creates a new log message
  349. *
  350. * @param {...any} args - anything to be included in the log message, the first argument is the type of log
  351. */
  352. log(...args) {
  353. const _arguments = Array.from(args);
  354. const type = _arguments[0];
  355. if (config.debug && config.debug.stationIssue === true && type === "STATION_ISSUE") {
  356. this.moduleManager.debugLogs.stationIssue.push(_arguments);
  357. return;
  358. }
  359. if (this.logRules.hideType.indexOf(type) !== -1) return;
  360. _arguments.splice(0, 1);
  361. const start = `|${this.name.toUpperCase()}|`;
  362. const numberOfSpacesNeeded = 20 - start.length;
  363. _arguments.unshift(`${start}${Array(numberOfSpacesNeeded).join(" ")}`);
  364. if (this.logRules.blacklistedTerms.some(blacklistedTerm => _arguments.join().indexOf(blacklistedTerm) !== -1))
  365. return;
  366. if (type === "INFO" || type === "SUCCESS") {
  367. _arguments[0] += "\x1b[36m";
  368. _arguments.push("\x1b[0m");
  369. console.log.apply(null, _arguments);
  370. } else if (type === "ERROR") {
  371. _arguments[0] += "\x1b[31m";
  372. _arguments.push("\x1b[0m");
  373. console.error.apply(null, _arguments);
  374. }
  375. }
  376. /**
  377. * Sets up each job with the statistics service (includes mean average for job completion)
  378. */
  379. registerJobs() {
  380. let props = [];
  381. let obj = this;
  382. do {
  383. props = props.concat(Object.getOwnPropertyNames(obj));
  384. // eslint-disable-next-line no-cond-assign
  385. } while ((obj = Object.getPrototypeOf(obj)));
  386. const jobNames = props.sort().filter(prop => typeof this[prop] === "function" && prop === prop.toUpperCase());
  387. jobNames.forEach(jobName => {
  388. this.jobStatistics[jobName] = {
  389. successful: 0,
  390. failed: 0,
  391. total: 0,
  392. averageTiming: new MovingAverageCalculator()
  393. };
  394. });
  395. }
  396. /**
  397. * Runs a job
  398. *
  399. * @param {string} name - the name of the job e.g. GET_PLAYLIST
  400. * @param {object} payload - any expected payload for the job itself
  401. * @param {object} parentJob - the parent job, if any
  402. * @param {number} priority - custom priority. Optional.
  403. * @param {object} options - custom options e.g. isQuiet. Optional.
  404. * @returns {Promise} - returns a promise
  405. */
  406. runJob(name, payload, parentJob, priority, options) {
  407. /** Allows for any combination of optional parameters (parentJob, priority, options) */
  408. let _options;
  409. let _priority;
  410. let _parentJob;
  411. if (parentJob) {
  412. if (typeof parentJob === "object")
  413. if (!parentJob.name) _options = parentJob;
  414. else _parentJob = parentJob;
  415. else if (typeof parentJob === "number") _priority = parentJob;
  416. }
  417. if (options) {
  418. if (typeof options === "object")
  419. if (options.name) _parentJob = options;
  420. else _options = options;
  421. if (typeof options === "number") _priority = options;
  422. }
  423. if (priority && typeof priority === "object") {
  424. if (!priority.name) _options = priority;
  425. else _parentJob = priority;
  426. } else _priority = priority;
  427. if (!_options) _options = { isQuiet: false };
  428. const deferredPromise = new DeferredPromise();
  429. const job = new Job(name, payload, deferredPromise, this, _parentJob);
  430. this.log("INFO", `Queuing job ${name} (${job.toString()})`);
  431. if (_parentJob) {
  432. _parentJob.addChildJob(job);
  433. if (_parentJob.status === "RUNNING") {
  434. this.log(
  435. "INFO",
  436. `Pausing job ${_parentJob.name} (${_parentJob.toString()}) since a child job has to run first`
  437. );
  438. _parentJob.setStatus("WAITING_ON_CHILD_JOB");
  439. _parentJob.module.jobQueue.pauseRunningJob(_parentJob);
  440. // console.log(111, _parentJob.module.jobQueue.length());
  441. // console.log(
  442. // 222,
  443. // _parentJob.module.jobQueue.workersList().map(data => data.data.job)
  444. // );
  445. } else {
  446. this.log(
  447. "INFO",
  448. `Not pausing job ${_parentJob.name} (${_parentJob.toString()}) since it's already paused`
  449. );
  450. }
  451. }
  452. // console.log(this);
  453. // console.log(321, _parentJob);
  454. job.setStatus("QUEUED");
  455. // if (options.bypassQueue) this._runJob(job, options, () => {});
  456. // else {
  457. let calculatedPriority = null;
  458. if (_priority) calculatedPriority = _priority;
  459. else if (this.priorities[name]) calculatedPriority = this.priorities[name];
  460. else if (_parentJob) calculatedPriority = _parentJob.task.priority;
  461. else calculatedPriority = 10;
  462. this.jobQueue.push(job, _options, calculatedPriority);
  463. if (
  464. config.debug &&
  465. config.debug.stationIssue === true &&
  466. config.debug.captureJobs &&
  467. config.debug.captureJobs.indexOf(name) !== -1
  468. ) {
  469. this.moduleManager.debugJobs.all.push({ job, _priority });
  470. }
  471. // }
  472. return deferredPromise.promise;
  473. }
  474. /**
  475. * UNKNOWN
  476. *
  477. * @param {object} moduleManager - UNKNOWN
  478. */
  479. setModuleManager(moduleManager) {
  480. this.moduleManager = moduleManager;
  481. }
  482. /**
  483. * Actually runs the job? UNKNOWN
  484. *
  485. * @param {object} job - object containing details of the job
  486. * @param {string} job.name - the name of the job e.g. GET_PLAYLIST
  487. * @param {string} job.payload - any expected payload for the job itself
  488. * @param {Promise} job.onFinish - deferred promise when the job is complete
  489. * @param {object} options - custom options e.g. isQuiet. Optional.
  490. * @returns {Promise} - returns a promise
  491. */
  492. _runJob(job, options) {
  493. if (!options.isQuiet) this.log("INFO", `Running job ${job.name} (${job.toString()})`);
  494. return new Promise(resolve => {
  495. const startTime = Date.now();
  496. const previousStatus = job.status;
  497. job.setStatus("RUNNING");
  498. this.moduleManager.jobManager.addJob(job);
  499. if (previousStatus === "QUEUED") {
  500. if (!options.isQuiet) this.log("INFO", `Job ${job.name} (${job.toString()}) is queued, so calling it`);
  501. this[job.name]
  502. .apply(job, [job.payload])
  503. .then(response => {
  504. if (!options.isQuiet) this.log("INFO", `Ran job ${job.name} (${job.toString()}) successfully`);
  505. job.setStatus("FINISHED");
  506. job.setResponse(response);
  507. this.jobStatistics[job.name].successful += 1;
  508. job.setResponseType("RESOLVE");
  509. if (
  510. config.debug &&
  511. config.debug.stationIssue === true &&
  512. config.debug.captureJobs &&
  513. config.debug.captureJobs.indexOf(job.name) !== -1
  514. ) {
  515. this.moduleManager.debugJobs.completed.push({
  516. status: "success",
  517. job,
  518. priority: job.task.priority,
  519. response
  520. });
  521. }
  522. // job.onFinish.resolve(response);
  523. })
  524. .catch(error => {
  525. this.log("INFO", `Running job ${job.name} (${job.toString()}) failed`);
  526. job.setStatus("FINISHED");
  527. job.setResponse(error);
  528. job.setResponseType("REJECT");
  529. this.jobStatistics[job.name].failed += 1;
  530. if (
  531. config.debug &&
  532. config.debug.stationIssue === true &&
  533. config.debug.captureJobs &&
  534. config.debug.captureJobs.indexOf(job.name) !== -1
  535. ) {
  536. this.moduleManager.debugJobs.completed.push({
  537. status: "error",
  538. job,
  539. error
  540. });
  541. }
  542. // job.onFinish.reject(error);
  543. })
  544. .finally(() => {
  545. const endTime = Date.now();
  546. const executionTime = endTime - startTime;
  547. this.jobStatistics[job.name].total += 1;
  548. this.jobStatistics[job.name].averageTiming.update(executionTime);
  549. this.moduleManager.jobManager.removeJob(job);
  550. job.cleanup();
  551. if (!job.parentJob) {
  552. if (job.responseType === "RESOLVE") {
  553. job.onFinish.resolve(job.response);
  554. job.responseType = "RESOLVED";
  555. } else if (job.responseType === "REJECT") {
  556. job.onFinish.reject(job.response);
  557. job.responseType = "REJECTED";
  558. }
  559. } else if (
  560. job.parentJob &&
  561. job.parentJob.childJobs.find(childJob => childJob.status !== "FINISHED") === undefined
  562. ) {
  563. if (job.parentJob.status !== "WAITING_ON_CHILD_JOB") {
  564. this.log(
  565. "ERROR",
  566. `Job ${
  567. job.parentJob.name
  568. } (${job.parentJob.toString()}) had a child job complete even though it is not waiting on a child job. This should never happen.`
  569. );
  570. } else {
  571. job.parentJob.setStatus("REQUEUED");
  572. job.parentJob.module.jobQueue.resumeRunningJob(job.parentJob);
  573. }
  574. }
  575. resolve();
  576. });
  577. } else {
  578. this.log(
  579. "INFO",
  580. `Job ${job.name} (${job.toString()}) is re-queued, so resolving/rejecting all child jobs.`
  581. );
  582. job.childJobs.forEach(childJob => {
  583. if (childJob.responseType === "RESOLVE") {
  584. childJob.onFinish.resolve(childJob.response);
  585. childJob.responseType = "RESOLVED";
  586. } else if (childJob.responseType === "REJECT") {
  587. childJob.onFinish.reject(childJob.response);
  588. childJob.responseType = "REJECTED";
  589. }
  590. });
  591. }
  592. });
  593. }
  594. }