core.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. import config from "config";
  2. class DeferredPromise {
  3. // eslint-disable-next-line require-jsdoc
  4. constructor() {
  5. this.promise = new Promise((resolve, reject) => {
  6. this.reject = reject;
  7. this.resolve = resolve;
  8. });
  9. }
  10. }
  11. class QueueTask {
  12. // eslint-disable-next-line require-jsdoc
  13. constructor(job, priority) {
  14. this.job = job;
  15. this.priority = priority;
  16. }
  17. }
  18. class Queue {
  19. // eslint-disable-next-line require-jsdoc
  20. constructor(handleTaskFunction, concurrency) {
  21. this.handleTaskFunction = handleTaskFunction;
  22. this.concurrency = concurrency;
  23. this.queue = [];
  24. this.runningTasks = [];
  25. this.pausedTasks = [];
  26. this.paused = false;
  27. }
  28. /**
  29. * Pauses the queue, meaning no new jobs can be started. Jobs can still be added to the queue, and already running tasks won't be paused.
  30. */
  31. pause() {
  32. this.paused = true;
  33. }
  34. /**
  35. * Resumes the queue.
  36. */
  37. resume() {
  38. this.paused = false;
  39. this._handleQueue();
  40. }
  41. /**
  42. * Returns the amount of jobs in the queue.
  43. *
  44. * @returns {number} - amount of jobs in queue
  45. */
  46. lengthQueue() {
  47. return this.queue.length;
  48. }
  49. /**
  50. * Returns the amount of running jobs.
  51. *
  52. * @returns {number} - amount of running jobs
  53. */
  54. lengthRunning() {
  55. return this.runningTasks.length;
  56. }
  57. /**
  58. * Adds a job to the queue, with a given priority.
  59. *
  60. * @param {object} job - the job that is to be added
  61. * @param {number} priority - the priority of the to be added job
  62. */
  63. push(job, priority) {
  64. this.queue.push(new QueueTask(job, priority));
  65. setTimeout(() => {
  66. this._handleQueue();
  67. }, 0);
  68. }
  69. /**
  70. * Removes a job currently running from the queue.
  71. *
  72. * @param {object} job - the job to be removed
  73. */
  74. removeRunningJob(job) {
  75. this.runningTasks.remove(this.runningTasks.find(task => task.job.toString() === job.toString()));
  76. }
  77. /**
  78. * Pauses a job currently running from the queue.
  79. *
  80. * @param {object} job - the job to be pauses
  81. */
  82. pauseRunningJob(job) {
  83. const task = this.runningTasks.find(task => task.job.toString() === job.toString());
  84. this.runningTasks.remove(task);
  85. this.pausedTasks.push(task);
  86. }
  87. /**
  88. * Resumes a job currently paused, adding the job back to the front of the queue
  89. *
  90. * @param {object} job - the job to be pauses
  91. */
  92. resumeRunningJob(job) {
  93. const task = this.pausedTasks.find(task => task.job.toString() === job.toString());
  94. this.pausedTasks.remove(task);
  95. this.queue.unshift(task);
  96. setTimeout(() => {
  97. this._handleQueue();
  98. }, 0);
  99. }
  100. /**
  101. * Check if there's room for a job to be processed, and if there is, run it.
  102. */
  103. _handleQueue() {
  104. if (!this.paused && this.runningTasks.length < this.concurrency && this.queue.length > 0) {
  105. const task = this.queue.reduce((a, b) => (a.priority < b.priority ? b : a));
  106. this.queue.remove(task);
  107. this.runningTasks.push(task);
  108. this._handleTask(task);
  109. setTimeout(() => {
  110. this._handleQueue();
  111. }, 0);
  112. }
  113. }
  114. /**
  115. * Handles a task, calling the handleTaskFunction provided in the constructor
  116. *
  117. * @param {object} task - the task to be handled
  118. */
  119. _handleTask(task) {
  120. this.handleTaskFunction(task.job).finally(() => {
  121. this.runningTasks.remove(task);
  122. this._handleQueue();
  123. });
  124. }
  125. }
  126. class Job {
  127. // eslint-disable-next-line require-jsdoc
  128. constructor(name, payload, onFinish, module, parentJob) {
  129. this.name = name;
  130. this.payload = payload;
  131. this.response = null;
  132. this.responseType = null;
  133. this.onFinish = onFinish;
  134. this.module = module;
  135. this.parentJob = parentJob;
  136. this.childJobs = [];
  137. /* eslint-disable no-bitwise, eqeqeq */
  138. this.uniqueId = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, c => {
  139. const r = (Math.random() * 16) | 0;
  140. const v = c == "x" ? r : (r & 0x3) | 0x8;
  141. return v.toString(16);
  142. });
  143. this.status = "INITIALIZED";
  144. }
  145. /**
  146. * Adds a child job to this job
  147. *
  148. * @param {object} childJob - the child job
  149. */
  150. addChildJob(childJob) {
  151. this.childJobs.push(childJob);
  152. }
  153. /**
  154. * Sets the job status
  155. *
  156. * @param {string} status - the new status
  157. */
  158. setStatus(status) {
  159. // console.log(`Job ${this.toString()} has changed status from ${this.status} to ${status}`);
  160. this.status = status;
  161. }
  162. /**
  163. * Returns the UUID of the job, allowing you to compare jobs with toString
  164. *
  165. * @returns {string} - the job's UUID/uniqueId
  166. */
  167. toString() {
  168. return this.uniqueId;
  169. }
  170. /**
  171. * Sets the response that will be provided to the onFinish DeferredPromise resolve/reject function, as soon as the job is done if it has no parent, or when the parent job is resumed
  172. *
  173. * @param {object} response - the response
  174. */
  175. setResponse(response) {
  176. this.response = response;
  177. }
  178. /**
  179. * Sets the response type that is paired with the response. If it is RESOLVE/REJECT, then it will resolve/reject with the response. If it is RESOLVED/REJECTED, then it has already resolved/rejected with the response.
  180. *
  181. * @param {string} responseType - the response type, so RESOLVE/REJECT/RESOLVED/REJECTED
  182. */
  183. setResponseType(responseType) {
  184. this.responseType = responseType;
  185. }
  186. }
  187. class MovingAverageCalculator {
  188. // eslint-disable-next-line require-jsdoc
  189. constructor() {
  190. this.count = 0;
  191. this._mean = 0;
  192. }
  193. /**
  194. * Updates the mean average
  195. *
  196. * @param {number} newValue - the new time it took to complete a job
  197. */
  198. update(newValue) {
  199. this.count += 1;
  200. const differential = (newValue - this._mean) / this.count;
  201. this._mean += differential;
  202. }
  203. /**
  204. * Returns the mean average
  205. *
  206. * @returns {number} - returns the mean average
  207. */
  208. get mean() {
  209. this.validate();
  210. return this._mean;
  211. }
  212. /**
  213. * Checks that the mean is valid
  214. */
  215. validate() {
  216. if (this.count === 0) throw new Error("Mean is undefined");
  217. }
  218. }
  219. export default class CoreClass {
  220. // eslint-disable-next-line require-jsdoc
  221. constructor(name) {
  222. this.name = name;
  223. this.status = "UNINITIALIZED";
  224. // this.log("Core constructor");
  225. this.jobQueue = new Queue(job => this._runJob(job), 10);
  226. this.jobQueue.pause();
  227. this.runningJobs = [];
  228. this.priorities = {};
  229. this.stage = 0;
  230. this.jobStatistics = {};
  231. this.registerJobs();
  232. }
  233. /**
  234. * Sets the status of a module
  235. *
  236. * @param {string} status - the new status of a module
  237. */
  238. setStatus(status) {
  239. this.status = status;
  240. this.log("INFO", `Status changed to: ${status}`);
  241. if (this.status === "READY") this.jobQueue.resume();
  242. else if (this.status === "FAIL" || this.status === "LOCKDOWN") this.jobQueue.pause();
  243. }
  244. /**
  245. * Returns the status of a module
  246. *
  247. * @returns {string} - the status of a module
  248. */
  249. getStatus() {
  250. return this.status;
  251. }
  252. /**
  253. * Changes the current stage of a module
  254. *
  255. * @param {string} stage - the new stage of a module
  256. */
  257. setStage(stage) {
  258. this.stage = stage;
  259. }
  260. /**
  261. * Returns the current stage of a module
  262. *
  263. * @returns {string} - the current stage of a module
  264. */
  265. getStage() {
  266. return this.stage;
  267. }
  268. /**
  269. * Initialises a module and handles initialise successes and failures
  270. */
  271. _initialize() {
  272. this.setStatus("INITIALIZING");
  273. this.initialize()
  274. .then(() => {
  275. this.setStatus("READY");
  276. this.moduleManager.onInitialize(this);
  277. })
  278. .catch(err => {
  279. console.error(err);
  280. this.setStatus("FAILED");
  281. this.moduleManager.onFail(this);
  282. });
  283. }
  284. /**
  285. * Creates a new log message
  286. *
  287. * @param {...any} args - anything to be included in the log message, the first argument is the type of log
  288. */
  289. log(...args) {
  290. const _arguments = Array.from(args);
  291. const type = _arguments[0];
  292. if (config.debug && config.debug.stationIssue === true && type === "STATION_ISSUE") {
  293. this.moduleManager.debugLogs.stationIssue.push(_arguments);
  294. return;
  295. }
  296. _arguments.splice(0, 1);
  297. const start = `|${this.name.toUpperCase()}|`;
  298. const numberOfSpacesNeeded = 20 - start.length;
  299. _arguments.unshift(`${start}${Array(numberOfSpacesNeeded).join(" ")}`);
  300. if (type === "INFO") {
  301. _arguments[0] += "\x1b[36m";
  302. _arguments.push("\x1b[0m");
  303. console.log.apply(null, _arguments);
  304. } else if (type === "ERROR") {
  305. _arguments[0] += "\x1b[31m";
  306. _arguments.push("\x1b[0m");
  307. console.error.apply(null, _arguments);
  308. }
  309. }
  310. /**
  311. * Sets up each job with the statistics service (includes mean average for job completion)
  312. */
  313. registerJobs() {
  314. let props = [];
  315. let obj = this;
  316. do {
  317. props = props.concat(Object.getOwnPropertyNames(obj));
  318. // eslint-disable-next-line no-cond-assign
  319. } while ((obj = Object.getPrototypeOf(obj)));
  320. const jobNames = props.sort().filter(prop => typeof this[prop] === "function" && prop === prop.toUpperCase());
  321. jobNames.forEach(jobName => {
  322. this.jobStatistics[jobName] = {
  323. successful: 0,
  324. failed: 0,
  325. total: 0,
  326. averageTiming: new MovingAverageCalculator()
  327. };
  328. });
  329. }
  330. /**
  331. * Runs a job
  332. *
  333. * @param {string} name - the name of the job e.g. GET_PLAYLIST
  334. * @param {object} payload - any expected payload for the job itself
  335. * @param {object} parentJob - the parent job, if any
  336. * @returns {Promise} - returns a promise
  337. */
  338. runJob(name, payload, parentJob) {
  339. const deferredPromise = new DeferredPromise();
  340. const job = new Job(name, payload, deferredPromise, this, parentJob);
  341. this.log("INFO", `Queuing job ${name} (${job.toString()})`);
  342. if (parentJob) {
  343. this.log(
  344. "INFO",
  345. `Pausing job ${parentJob.name} (${parentJob.toString()}) since a child job has to run first`
  346. );
  347. parentJob.addChildJob(job);
  348. parentJob.setStatus("WAITING_ON_CHILD_JOB");
  349. parentJob.module.jobQueue.pauseRunningJob(parentJob);
  350. // console.log(111, parentJob.module.jobQueue.length());
  351. // console.log(
  352. // 222,
  353. // parentJob.module.jobQueue.workersList().map(data => data.data.job)
  354. // );
  355. }
  356. // console.log(this);
  357. // console.log(321, parentJob);
  358. if (
  359. config.debug &&
  360. config.debug.stationIssue === true &&
  361. config.debug.captureJobs &&
  362. config.debug.captureJobs.indexOf(name) !== -1
  363. ) {
  364. this.moduleManager.debugJobs.all.push(job);
  365. }
  366. job.setStatus("QUEUED");
  367. // if (options.bypassQueue) this._runJob(job, options, () => {});
  368. // else {
  369. const priority = this.priorities[name] ? this.priorities[name] : 10;
  370. this.jobQueue.push(job, priority);
  371. // }
  372. return deferredPromise.promise;
  373. }
  374. /**
  375. * UNKNOWN
  376. *
  377. * @param {object} moduleManager - UNKNOWN
  378. */
  379. setModuleManager(moduleManager) {
  380. this.moduleManager = moduleManager;
  381. }
  382. /**
  383. * Actually runs the job? UNKNOWN
  384. *
  385. * @param {object} job - object containing details of the job
  386. * @param {string} job.name - the name of the job e.g. GET_PLAYLIST
  387. * @param {string} job.payload - any expected payload for the job itself
  388. * @param {Promise} job.onFinish - deferred promise when the job is complete
  389. * @returns {Promise} - returns a promise
  390. */
  391. _runJob(job) {
  392. this.log("INFO", `Running job ${job.name} (${job.toString()})`);
  393. return new Promise(resolve => {
  394. const startTime = Date.now();
  395. const previousStatus = job.status;
  396. job.setStatus("RUNNING");
  397. this.runningJobs.push(job);
  398. if (previousStatus === "QUEUED") {
  399. this.log("INFO", `Job ${job.name} (${job.toString()}) is queued, so calling it`);
  400. this[job.name]
  401. .apply(job, [job.payload])
  402. .then(response => {
  403. // if (!options.isQuiet)
  404. this.log("INFO", `Ran job ${job.name} (${job.toString()}) successfully`);
  405. job.setStatus("FINISHED");
  406. job.setResponse(response);
  407. this.jobStatistics[job.name].successful += 1;
  408. job.setResponseType("RESOLVE");
  409. if (
  410. config.debug &&
  411. config.debug.stationIssue === true &&
  412. config.debug.captureJobs &&
  413. config.debug.captureJobs.indexOf(job.name) !== -1
  414. ) {
  415. this.moduleManager.debugJobs.completed.push({
  416. status: "success",
  417. job,
  418. response
  419. });
  420. }
  421. // job.onFinish.resolve(response);
  422. })
  423. .catch(error => {
  424. this.log("INFO", `Running job ${job.name} (${job.toString()}) failed`);
  425. job.setStatus("FINISHED");
  426. job.setResponse(error);
  427. job.setResponseType("REJECT");
  428. this.jobStatistics[job.name].failed += 1;
  429. if (
  430. config.debug &&
  431. config.debug.stationIssue === true &&
  432. config.debug.captureJobs &&
  433. config.debug.captureJobs.indexOf(job.name) !== -1
  434. ) {
  435. this.moduleManager.debugJobs.completed.push({
  436. status: "error",
  437. job,
  438. error
  439. });
  440. }
  441. // job.onFinish.reject(error);
  442. })
  443. .finally(() => {
  444. const endTime = Date.now();
  445. const executionTime = endTime - startTime;
  446. this.jobStatistics[job.name].total += 1;
  447. this.jobStatistics[job.name].averageTiming.update(executionTime);
  448. this.runningJobs.splice(this.runningJobs.indexOf(job), 1);
  449. if (!job.parentJob) {
  450. if (job.responseType === "RESOLVE") {
  451. job.onFinish.resolve(job.response);
  452. job.responseType = "RESOLVED";
  453. } else if (job.responseType === "REJECT") {
  454. job.onFinish.reject(job.response);
  455. job.responseType = "REJECTED";
  456. }
  457. } else if (
  458. job.parentJob &&
  459. job.parentJob.childJobs.find(childJob => childJob.status !== "FINISHED") === undefined
  460. ) {
  461. this.log(
  462. "INFO",
  463. `Requeing/resuming job ${
  464. job.parentJob.name
  465. } (${job.parentJob.toString()}) since all child jobs are complete.`
  466. );
  467. job.parentJob.setStatus("REQUEUED");
  468. job.parentJob.module.jobQueue.resumeRunningJob(job.parentJob);
  469. }
  470. resolve();
  471. });
  472. } else {
  473. this.log(
  474. "INFO",
  475. `Job ${job.name} (${job.toString()}) is re-queued, so resolving/rejecting all child jobs.`
  476. );
  477. job.childJobs.forEach(childJob => {
  478. if (childJob.responseType === "RESOLVE") {
  479. childJob.onFinish.resolve(childJob.response);
  480. childJob.responseType = "RESOLVED";
  481. } else if (childJob.responseType === "REJECT") {
  482. childJob.onFinish.reject(childJob.response);
  483. childJob.responseType = "REJECTED";
  484. }
  485. });
  486. }
  487. });
  488. }
  489. }