core.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584
  1. import config from "config";
  2. class DeferredPromise {
  3. // eslint-disable-next-line require-jsdoc
  4. constructor() {
  5. this.promise = new Promise((resolve, reject) => {
  6. this.reject = reject;
  7. this.resolve = resolve;
  8. });
  9. }
  10. }
  11. class QueueTask {
  12. // eslint-disable-next-line require-jsdoc
  13. constructor(job, priority) {
  14. this.job = job;
  15. this.priority = priority;
  16. this.job.setTask(this);
  17. }
  18. }
  19. class Queue {
  20. // eslint-disable-next-line require-jsdoc
  21. constructor(handleTaskFunction, concurrency) {
  22. this.handleTaskFunction = handleTaskFunction;
  23. this.concurrency = concurrency;
  24. this.queue = [];
  25. this.runningTasks = [];
  26. this.pausedTasks = [];
  27. this.paused = false;
  28. }
  29. /**
  30. * Pauses the queue, meaning no new jobs can be started. Jobs can still be added to the queue, and already running tasks won't be paused.
  31. */
  32. pause() {
  33. this.paused = true;
  34. }
  35. /**
  36. * Resumes the queue.
  37. */
  38. resume() {
  39. this.paused = false;
  40. setTimeout(() => {
  41. this._handleQueue();
  42. }, 0);
  43. }
  44. /**
  45. * Returns the amount of jobs in the queue.
  46. *
  47. * @returns {number} - amount of jobs in queue
  48. */
  49. lengthQueue() {
  50. return this.queue.length;
  51. }
  52. /**
  53. * Returns the amount of running jobs.
  54. *
  55. * @returns {number} - amount of running jobs
  56. */
  57. lengthRunning() {
  58. return this.runningTasks.length;
  59. }
  60. /**
  61. * Returns the amount of running jobs.
  62. *
  63. * @returns {number} - amount of running jobs
  64. */
  65. lengthPaused() {
  66. return this.pausedTasks.length;
  67. }
  68. /**
  69. * Adds a job to the queue, with a given priority.
  70. *
  71. * @param {object} job - the job that is to be added
  72. * @param {number} priority - the priority of the to be added job
  73. */
  74. push(job, priority) {
  75. this.queue.push(new QueueTask(job, priority));
  76. setTimeout(() => {
  77. this._handleQueue();
  78. }, 0);
  79. }
  80. /**
  81. * Removes a job currently running from the queue.
  82. *
  83. * @param {object} job - the job to be removed
  84. */
  85. removeRunningJob(job) {
  86. this.runningTasks.remove(this.runningTasks.find(task => task.job.toString() === job.toString()));
  87. }
  88. /**
  89. * Pauses a job currently running from the queue.
  90. *
  91. * @param {object} job - the job to be pauses
  92. */
  93. pauseRunningJob(job) {
  94. const task = this.runningTasks.find(task => task.job.toString() === job.toString());
  95. if (!task) {
  96. console.log(
  97. `Attempted to pause job ${job.name} (${job.toString()}), but couldn't find it in running tasks.`
  98. );
  99. return;
  100. }
  101. this.runningTasks.remove(task);
  102. this.pausedTasks.push(task);
  103. }
  104. /**
  105. * Resumes a job currently paused, adding the job back to the front of the queue
  106. *
  107. * @param {object} job - the job to be pauses
  108. */
  109. resumeRunningJob(job) {
  110. const task = this.pausedTasks.find(task => task.job.toString() === job.toString());
  111. if (!task) {
  112. console.log(
  113. `Attempted to resume job ${job.name} (${job.toString()}), but couldn't find it in paused tasks.`
  114. );
  115. return;
  116. }
  117. this.pausedTasks.remove(task);
  118. this.queue.unshift(task);
  119. setTimeout(() => {
  120. this._handleQueue();
  121. }, 0);
  122. }
  123. /**
  124. * Check if there's room for a job to be processed, and if there is, run it.
  125. */
  126. _handleQueue() {
  127. if (this.queue.length > 0) {
  128. const task = this.queue.reduce((a, b) => (a.priority < b.priority ? a : b));
  129. // console.log(`First task: `, task);
  130. if (task) {
  131. if ((!this.paused && this.runningTasks.length < this.concurrency) || task.priority === -1) {
  132. this.queue.remove(task);
  133. this.runningTasks.push(task);
  134. this._handleTask(task);
  135. setTimeout(() => {
  136. this._handleQueue();
  137. }, 0);
  138. }
  139. }
  140. }
  141. }
  142. /**
  143. * Handles a task, calling the handleTaskFunction provided in the constructor
  144. *
  145. * @param {object} task - the task to be handled
  146. */
  147. _handleTask(task) {
  148. this.handleTaskFunction(task.job).finally(() => {
  149. this.runningTasks.remove(task);
  150. this._handleQueue();
  151. });
  152. }
  153. }
  154. class Job {
  155. // eslint-disable-next-line require-jsdoc
  156. constructor(name, payload, onFinish, module, parentJob) {
  157. this.name = name;
  158. this.payload = payload;
  159. this.response = null;
  160. this.responseType = null;
  161. this.onFinish = onFinish;
  162. this.module = module;
  163. this.parentJob = parentJob;
  164. this.childJobs = [];
  165. /* eslint-disable no-bitwise, eqeqeq */
  166. this.uniqueId = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, c => {
  167. const r = (Math.random() * 16) | 0;
  168. const v = c == "x" ? r : (r & 0x3) | 0x8;
  169. return v.toString(16);
  170. });
  171. this.status = "INITIALIZED";
  172. this.task = null;
  173. }
  174. /**
  175. * Adds a child job to this job
  176. *
  177. * @param {object} childJob - the child job
  178. */
  179. addChildJob(childJob) {
  180. this.childJobs.push(childJob);
  181. }
  182. /**
  183. * Sets the job status
  184. *
  185. * @param {string} status - the new status
  186. */
  187. setStatus(status) {
  188. // console.log(`Job ${this.toString()} has changed status from ${this.status} to ${status}`);
  189. this.status = status;
  190. }
  191. setTask(task) {
  192. this.task = task;
  193. }
  194. /**
  195. * Returns the UUID of the job, allowing you to compare jobs with toString
  196. *
  197. * @returns {string} - the job's UUID/uniqueId
  198. */
  199. toString() {
  200. return this.uniqueId;
  201. }
  202. /**
  203. * Sets the response that will be provided to the onFinish DeferredPromise resolve/reject function, as soon as the job is done if it has no parent, or when the parent job is resumed
  204. *
  205. * @param {object} response - the response
  206. */
  207. setResponse(response) {
  208. this.response = response;
  209. }
  210. /**
  211. * Sets the response type that is paired with the response. If it is RESOLVE/REJECT, then it will resolve/reject with the response. If it is RESOLVED/REJECTED, then it has already resolved/rejected with the response.
  212. *
  213. * @param {string} responseType - the response type, so RESOLVE/REJECT/RESOLVED/REJECTED
  214. */
  215. setResponseType(responseType) {
  216. this.responseType = responseType;
  217. }
  218. }
  219. class MovingAverageCalculator {
  220. // eslint-disable-next-line require-jsdoc
  221. constructor() {
  222. this.count = 0;
  223. this._mean = 0;
  224. }
  225. /**
  226. * Updates the mean average
  227. *
  228. * @param {number} newValue - the new time it took to complete a job
  229. */
  230. update(newValue) {
  231. this.count += 1;
  232. const differential = (newValue - this._mean) / this.count;
  233. this._mean += differential;
  234. }
  235. /**
  236. * Returns the mean average
  237. *
  238. * @returns {number} - returns the mean average
  239. */
  240. get mean() {
  241. this.validate();
  242. return this._mean;
  243. }
  244. /**
  245. * Checks that the mean is valid
  246. */
  247. validate() {
  248. if (this.count === 0) throw new Error("Mean is undefined");
  249. }
  250. }
  251. export default class CoreClass {
  252. // eslint-disable-next-line require-jsdoc
  253. constructor(name) {
  254. this.name = name;
  255. this.status = "UNINITIALIZED";
  256. // this.log("Core constructor");
  257. this.jobQueue = new Queue(job => this._runJob(job), 10);
  258. this.jobQueue.pause();
  259. this.runningJobs = [];
  260. this.priorities = {};
  261. this.stage = 0;
  262. this.jobStatistics = {};
  263. this.registerJobs();
  264. }
  265. /**
  266. * Sets the status of a module
  267. *
  268. * @param {string} status - the new status of a module
  269. */
  270. setStatus(status) {
  271. this.status = status;
  272. this.log("INFO", `Status changed to: ${status}`);
  273. if (this.status === "READY") this.jobQueue.resume();
  274. else if (this.status === "FAIL" || this.status === "LOCKDOWN") this.jobQueue.pause();
  275. }
  276. /**
  277. * Returns the status of a module
  278. *
  279. * @returns {string} - the status of a module
  280. */
  281. getStatus() {
  282. return this.status;
  283. }
  284. /**
  285. * Changes the current stage of a module
  286. *
  287. * @param {string} stage - the new stage of a module
  288. */
  289. setStage(stage) {
  290. this.stage = stage;
  291. }
  292. /**
  293. * Returns the current stage of a module
  294. *
  295. * @returns {string} - the current stage of a module
  296. */
  297. getStage() {
  298. return this.stage;
  299. }
  300. /**
  301. * Initialises a module and handles initialise successes and failures
  302. */
  303. _initialize() {
  304. this.setStatus("INITIALIZING");
  305. this.initialize()
  306. .then(() => {
  307. this.setStatus("READY");
  308. this.moduleManager.onInitialize(this);
  309. })
  310. .catch(err => {
  311. console.error(err);
  312. this.setStatus("FAILED");
  313. this.moduleManager.onFail(this);
  314. });
  315. }
  316. /**
  317. * Creates a new log message
  318. *
  319. * @param {...any} args - anything to be included in the log message, the first argument is the type of log
  320. */
  321. log(...args) {
  322. const _arguments = Array.from(args);
  323. const type = _arguments[0];
  324. if (config.debug && config.debug.stationIssue === true && type === "STATION_ISSUE") {
  325. this.moduleManager.debugLogs.stationIssue.push(_arguments);
  326. return;
  327. }
  328. _arguments.splice(0, 1);
  329. const start = `|${this.name.toUpperCase()}|`;
  330. const numberOfSpacesNeeded = 20 - start.length;
  331. _arguments.unshift(`${start}${Array(numberOfSpacesNeeded).join(" ")}`);
  332. if (type === "INFO") {
  333. _arguments[0] += "\x1b[36m";
  334. _arguments.push("\x1b[0m");
  335. console.log.apply(null, _arguments);
  336. } else if (type === "ERROR") {
  337. _arguments[0] += "\x1b[31m";
  338. _arguments.push("\x1b[0m");
  339. console.error.apply(null, _arguments);
  340. }
  341. }
  342. /**
  343. * Sets up each job with the statistics service (includes mean average for job completion)
  344. */
  345. registerJobs() {
  346. let props = [];
  347. let obj = this;
  348. do {
  349. props = props.concat(Object.getOwnPropertyNames(obj));
  350. // eslint-disable-next-line no-cond-assign
  351. } while ((obj = Object.getPrototypeOf(obj)));
  352. const jobNames = props.sort().filter(prop => typeof this[prop] === "function" && prop === prop.toUpperCase());
  353. jobNames.forEach(jobName => {
  354. this.jobStatistics[jobName] = {
  355. successful: 0,
  356. failed: 0,
  357. total: 0,
  358. averageTiming: new MovingAverageCalculator()
  359. };
  360. });
  361. }
  362. /**
  363. * Runs a job
  364. *
  365. * @param {string} name - the name of the job e.g. GET_PLAYLIST
  366. * @param {object} payload - any expected payload for the job itself
  367. * @param {object} parentJob - the parent job, if any
  368. * @param {number} priority - custom priority. Optional.
  369. * @returns {Promise} - returns a promise
  370. */
  371. runJob(name, payload, parentJob, priority) {
  372. const deferredPromise = new DeferredPromise();
  373. const job = new Job(name, payload, deferredPromise, this, parentJob);
  374. this.log("INFO", `Queuing job ${name} (${job.toString()})`);
  375. if (parentJob) {
  376. parentJob.addChildJob(job);
  377. if (parentJob.status === "RUNNING") {
  378. this.log(
  379. "INFO",
  380. `Pausing job ${parentJob.name} (${parentJob.toString()}) since a child job has to run first`
  381. );
  382. parentJob.setStatus("WAITING_ON_CHILD_JOB");
  383. parentJob.module.jobQueue.pauseRunningJob(parentJob);
  384. // console.log(111, parentJob.module.jobQueue.length());
  385. // console.log(
  386. // 222,
  387. // parentJob.module.jobQueue.workersList().map(data => data.data.job)
  388. // );
  389. } else {
  390. this.log(
  391. "INFO",
  392. `Not pausing job ${parentJob.name} (${parentJob.toString()}) since it's already paused`
  393. );
  394. }
  395. }
  396. // console.log(this);
  397. // console.log(321, parentJob);
  398. if (
  399. config.debug &&
  400. config.debug.stationIssue === true &&
  401. config.debug.captureJobs &&
  402. config.debug.captureJobs.indexOf(name) !== -1
  403. ) {
  404. this.moduleManager.debugJobs.all.push(job);
  405. }
  406. job.setStatus("QUEUED");
  407. // if (options.bypassQueue) this._runJob(job, options, () => {});
  408. // else {
  409. const _priority = Math.min(
  410. priority || Infinity,
  411. parentJob ? parentJob.task.priority : Infinity,
  412. this.priorities[name] ? this.priorities[name] : 10
  413. );
  414. this.jobQueue.push(job, _priority);
  415. // }
  416. return deferredPromise.promise;
  417. }
  418. /**
  419. * UNKNOWN
  420. *
  421. * @param {object} moduleManager - UNKNOWN
  422. */
  423. setModuleManager(moduleManager) {
  424. this.moduleManager = moduleManager;
  425. }
  426. /**
  427. * Actually runs the job? UNKNOWN
  428. *
  429. * @param {object} job - object containing details of the job
  430. * @param {string} job.name - the name of the job e.g. GET_PLAYLIST
  431. * @param {string} job.payload - any expected payload for the job itself
  432. * @param {Promise} job.onFinish - deferred promise when the job is complete
  433. * @returns {Promise} - returns a promise
  434. */
  435. _runJob(job) {
  436. this.log("INFO", `Running job ${job.name} (${job.toString()})`);
  437. return new Promise(resolve => {
  438. const startTime = Date.now();
  439. const previousStatus = job.status;
  440. job.setStatus("RUNNING");
  441. this.runningJobs.push(job);
  442. if (previousStatus === "QUEUED") {
  443. this.log("INFO", `Job ${job.name} (${job.toString()}) is queued, so calling it`);
  444. this[job.name]
  445. .apply(job, [job.payload])
  446. .then(response => {
  447. // if (!options.isQuiet)
  448. this.log("INFO", `Ran job ${job.name} (${job.toString()}) successfully`);
  449. job.setStatus("FINISHED");
  450. job.setResponse(response);
  451. this.jobStatistics[job.name].successful += 1;
  452. job.setResponseType("RESOLVE");
  453. if (
  454. config.debug &&
  455. config.debug.stationIssue === true &&
  456. config.debug.captureJobs &&
  457. config.debug.captureJobs.indexOf(job.name) !== -1
  458. ) {
  459. this.moduleManager.debugJobs.completed.push({
  460. status: "success",
  461. job,
  462. response
  463. });
  464. }
  465. // job.onFinish.resolve(response);
  466. })
  467. .catch(error => {
  468. this.log("INFO", `Running job ${job.name} (${job.toString()}) failed`);
  469. job.setStatus("FINISHED");
  470. job.setResponse(error);
  471. job.setResponseType("REJECT");
  472. this.jobStatistics[job.name].failed += 1;
  473. if (
  474. config.debug &&
  475. config.debug.stationIssue === true &&
  476. config.debug.captureJobs &&
  477. config.debug.captureJobs.indexOf(job.name) !== -1
  478. ) {
  479. this.moduleManager.debugJobs.completed.push({
  480. status: "error",
  481. job,
  482. error
  483. });
  484. }
  485. // job.onFinish.reject(error);
  486. })
  487. .finally(() => {
  488. const endTime = Date.now();
  489. const executionTime = endTime - startTime;
  490. this.jobStatistics[job.name].total += 1;
  491. this.jobStatistics[job.name].averageTiming.update(executionTime);
  492. this.runningJobs.splice(this.runningJobs.indexOf(job), 1);
  493. if (!job.parentJob) {
  494. if (job.responseType === "RESOLVE") {
  495. job.onFinish.resolve(job.response);
  496. job.responseType = "RESOLVED";
  497. } else if (job.responseType === "REJECT") {
  498. job.onFinish.reject(job.response);
  499. job.responseType = "REJECTED";
  500. }
  501. } else if (
  502. job.parentJob &&
  503. job.parentJob.childJobs.find(childJob => childJob.status !== "FINISHED") === undefined
  504. ) {
  505. this.log(
  506. "INFO",
  507. `Requeing/resuming job ${
  508. job.parentJob.name
  509. } (${job.parentJob.toString()}) since all child jobs are complete.`
  510. );
  511. job.parentJob.setStatus("REQUEUED");
  512. job.parentJob.module.jobQueue.resumeRunningJob(job.parentJob);
  513. }
  514. resolve();
  515. });
  516. } else {
  517. this.log(
  518. "INFO",
  519. `Job ${job.name} (${job.toString()}) is re-queued, so resolving/rejecting all child jobs.`
  520. );
  521. job.childJobs.forEach(childJob => {
  522. if (childJob.responseType === "RESOLVE") {
  523. childJob.onFinish.resolve(childJob.response);
  524. childJob.responseType = "RESOLVED";
  525. } else if (childJob.responseType === "REJECT") {
  526. childJob.onFinish.reject(childJob.response);
  527. childJob.responseType = "REJECTED";
  528. }
  529. });
  530. }
  531. });
  532. }
  533. }