core.js 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434
  1. import async from "async";
  2. import config from "config";
  3. class DeferredPromise {
  4. // eslint-disable-next-line require-jsdoc
  5. constructor() {
  6. this.promise = new Promise((resolve, reject) => {
  7. this.reject = reject;
  8. this.resolve = resolve;
  9. });
  10. }
  11. }
  12. class QueueTask {
  13. // eslint-disable-next-line require-jsdoc
  14. constructor(job, priority) {
  15. this.job = job;
  16. this.priority = priority;
  17. }
  18. }
  19. class Queue {
  20. // eslint-disable-next-line require-jsdoc
  21. constructor(handleTaskFunction, concurrency) {
  22. this.handleTaskFunction = handleTaskFunction;
  23. this.concurrency = concurrency;
  24. this.queue = [];
  25. this.running = [];
  26. this.paused = false;
  27. }
  28. /**
  29. * Pauses the queue, meaning no new jobs can be started. Jobs can still be added to the queue, and already running tasks won't be paused.
  30. */
  31. pause() {
  32. this.paused = true;
  33. }
  34. /**
  35. * Resumes the queue.
  36. */
  37. resume() {
  38. this.paused = false;
  39. this._handleQueue();
  40. }
  41. /**
  42. * Returns the amount of jobs in the queue.
  43. *
  44. * @returns {number} - amount of jobs in queue
  45. */
  46. lengthQueue() {
  47. return this.queue.length;
  48. }
  49. /**
  50. * Returns the amount of running jobs.
  51. *
  52. * @returns {number} - amount of running jobs
  53. */
  54. lengthRunning() {
  55. return this.running.length;
  56. }
  57. /**
  58. * Adds a job to the queue, with a given priority.
  59. *
  60. * @param {object} job - the job that is to be added
  61. * @param {number} priority - the priority of the to be added job
  62. */
  63. push(job, priority) {
  64. this.queue.push(new QueueTask(job, priority));
  65. setTimeout(() => {
  66. this._handleQueue();
  67. }, 0);
  68. }
  69. /**
  70. * Removes a job currently running from the queue.
  71. *
  72. * @param {object} job - the job to be removed
  73. */
  74. removeRunningJob(job) {
  75. this.running.remove(this.running.find(task => task.job.toString() === job.toString()));
  76. }
  77. /**
  78. * Check if there's room for a job to be processed, and if there is, run it.
  79. */
  80. _handleQueue() {
  81. if (!this.paused && this.running.length < this.concurrency && this.queue.length > 0) {
  82. const task = this.queue.reduce((a, b) => (a.priority < b.priority ? b : a));
  83. this.queue.remove(task);
  84. this.running.push(task);
  85. this._handleTask(task);
  86. setTimeout(() => {
  87. this._handleQueue();
  88. }, 0);
  89. }
  90. }
  91. _handleTask(task) {
  92. this.handleTaskFunction(task.job).finally(() => {
  93. this.running.remove(task);
  94. this._handleQueue();
  95. });
  96. }
  97. }
  98. class Job {
  99. // eslint-disable-next-line require-jsdoc
  100. constructor(name, payload, onFinish, module, parentJob) {
  101. this.name = name;
  102. this.payload = payload;
  103. this.onFinish = onFinish;
  104. this.module = module;
  105. this.parentJob = parentJob;
  106. this.childJobs = [];
  107. /* eslint-disable no-bitwise, eqeqeq */
  108. this.uniqueId = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, c => {
  109. const r = (Math.random() * 16) | 0;
  110. const v = c == "x" ? r : (r & 0x3) | 0x8;
  111. return v.toString(16);
  112. });
  113. this.status = "INITIALIZED";
  114. }
  115. addChildJob(childJob) {
  116. this.childJobs.push(childJob);
  117. }
  118. setStatus(status) {
  119. this.status = status;
  120. }
  121. toString() {
  122. return this.uniqueId;
  123. }
  124. }
  125. class MovingAverageCalculator {
  126. // eslint-disable-next-line require-jsdoc
  127. constructor() {
  128. this.count = 0;
  129. this._mean = 0;
  130. }
  131. /**
  132. * Updates the mean average
  133. *
  134. * @param {number} newValue - the new time it took to complete a job
  135. */
  136. update(newValue) {
  137. this.count += 1;
  138. const differential = (newValue - this._mean) / this.count;
  139. this._mean += differential;
  140. }
  141. /**
  142. * Returns the mean average
  143. *
  144. * @returns {number} - returns the mean average
  145. */
  146. get mean() {
  147. this.validate();
  148. return this._mean;
  149. }
  150. /**
  151. * Checks that the mean is valid
  152. */
  153. validate() {
  154. if (this.count === 0) throw new Error("Mean is undefined");
  155. }
  156. }
  157. export default class CoreClass {
  158. // eslint-disable-next-line require-jsdoc
  159. constructor(name) {
  160. this.name = name;
  161. this.status = "UNINITIALIZED";
  162. // this.log("Core constructor");
  163. this.jobQueue = new Queue(job => this._runJob(job), 10);
  164. this.jobQueue.pause();
  165. this.runningJobs = [];
  166. this.priorities = {};
  167. this.stage = 0;
  168. this.jobStatistics = {};
  169. this.registerJobs();
  170. }
  171. /**
  172. * Sets the status of a module
  173. *
  174. * @param {string} status - the new status of a module
  175. */
  176. setStatus(status) {
  177. this.status = status;
  178. this.log("INFO", `Status changed to: ${status}`);
  179. if (this.status === "READY") this.jobQueue.resume();
  180. else if (this.status === "FAIL" || this.status === "LOCKDOWN") this.jobQueue.pause();
  181. }
  182. /**
  183. * Returns the status of a module
  184. *
  185. * @returns {string} - the status of a module
  186. */
  187. getStatus() {
  188. return this.status;
  189. }
  190. /**
  191. * Changes the current stage of a module
  192. *
  193. * @param {string} stage - the new stage of a module
  194. */
  195. setStage(stage) {
  196. this.stage = stage;
  197. }
  198. /**
  199. * Returns the current stage of a module
  200. *
  201. * @returns {string} - the current stage of a module
  202. */
  203. getStage() {
  204. return this.stage;
  205. }
  206. /**
  207. * Initialises a module and handles initialise successes and failures
  208. */
  209. _initialize() {
  210. this.setStatus("INITIALIZING");
  211. this.initialize()
  212. .then(() => {
  213. this.setStatus("READY");
  214. this.moduleManager.onInitialize(this);
  215. })
  216. .catch(err => {
  217. console.error(err);
  218. this.setStatus("FAILED");
  219. this.moduleManager.onFail(this);
  220. });
  221. }
  222. /**
  223. * Creates a new log message
  224. *
  225. * @param {...any} args - anything to be included in the log message, the first argument is the type of log
  226. */
  227. log(...args) {
  228. const _arguments = Array.from(args);
  229. const type = _arguments[0];
  230. if (config.debug && config.debug.stationIssue === true && type === "STATION_ISSUE") {
  231. this.moduleManager.debugLogs.stationIssue.push(_arguments);
  232. return;
  233. }
  234. _arguments.splice(0, 1);
  235. const start = `|${this.name.toUpperCase()}|`;
  236. const numberOfSpacesNeeded = 20 - start.length;
  237. _arguments.unshift(`${start}${Array(numberOfSpacesNeeded).join(" ")}`);
  238. if (type === "INFO") {
  239. _arguments[0] += "\x1b[36m";
  240. _arguments.push("\x1b[0m");
  241. console.log.apply(null, _arguments);
  242. } else if (type === "ERROR") {
  243. _arguments[0] += "\x1b[31m";
  244. _arguments.push("\x1b[0m");
  245. console.error.apply(null, _arguments);
  246. }
  247. }
  248. /**
  249. * Sets up each job with the statistics service (includes mean average for job completion)
  250. */
  251. registerJobs() {
  252. let props = [];
  253. let obj = this;
  254. do {
  255. props = props.concat(Object.getOwnPropertyNames(obj));
  256. // eslint-disable-next-line no-cond-assign
  257. } while ((obj = Object.getPrototypeOf(obj)));
  258. const jobNames = props.sort().filter(prop => typeof this[prop] === "function" && prop === prop.toUpperCase());
  259. jobNames.forEach(jobName => {
  260. this.jobStatistics[jobName] = {
  261. successful: 0,
  262. failed: 0,
  263. total: 0,
  264. averageTiming: new MovingAverageCalculator()
  265. };
  266. });
  267. }
  268. /**
  269. * Runs a job
  270. *
  271. * @param {string} name - the name of the job e.g. GET_PLAYLIST
  272. * @param {object} payload - any expected payload for the job itself
  273. * @param {object} parentJob - the parent job, if any
  274. * @returns {Promise} - returns a promise
  275. */
  276. runJob(name, payload, parentJob) {
  277. const deferredPromise = new DeferredPromise();
  278. const job = new Job(name, payload, deferredPromise, this, parentJob);
  279. this.log("INFO", `Queuing job ${name} (${job.toString()})`);
  280. if (parentJob) {
  281. this.log(
  282. "INFO",
  283. `Removing job ${
  284. parentJob.name
  285. } (${parentJob.toString()}) from running jobs since a child job has to run first`
  286. );
  287. parentJob.addChildJob(job);
  288. parentJob.setStatus("WAITING_ON_CHILD_JOB");
  289. parentJob.module.runningJobs.remove(job);
  290. parentJob.module.jobQueue.removeRunningJob(job);
  291. // console.log(111, parentJob.module.jobQueue.length());
  292. // console.log(
  293. // 222,
  294. // parentJob.module.jobQueue.workersList().map(data => data.data.job)
  295. // );
  296. }
  297. // console.log(this);
  298. // console.log(321, parentJob);
  299. if (
  300. config.debug &&
  301. config.debug.stationIssue === true &&
  302. config.debug.captureJobs &&
  303. config.debug.captureJobs.indexOf(name) !== -1
  304. ) {
  305. this.moduleManager.debugJobs.all.push(job);
  306. }
  307. job.setStatus("QUEUED");
  308. // if (options.bypassQueue) this._runJob(job, options, () => {});
  309. // else {
  310. const priority = this.priorities[name] ? this.priorities[name] : 10;
  311. this.jobQueue.push(job, priority);
  312. // }
  313. return deferredPromise.promise;
  314. }
  315. /**
  316. * UNKNOWN
  317. *
  318. * @param {object} moduleManager - UNKNOWN
  319. */
  320. setModuleManager(moduleManager) {
  321. this.moduleManager = moduleManager;
  322. }
  323. /**
  324. * Actually runs the job? UNKNOWN
  325. *
  326. * @param {object} job - object containing details of the job
  327. * @param {string} job.name - the name of the job e.g. GET_PLAYLIST
  328. * @param {string} job.payload - any expected payload for the job itself
  329. * @param {Promise} job.onFinish - deferred promise when the job is complete
  330. * @param {object} options - object containing any additional options for the job
  331. * @returns {Promise} - returns a promise
  332. */
  333. _runJob(job, options) {
  334. // if (!options.isQuiet)
  335. this.log("INFO", `Running job ${job.name} (${job.toString()})`);
  336. return new Promise((resolve, reject) => {
  337. const startTime = Date.now();
  338. job.setStatus("RUNNING");
  339. this.runningJobs.push(job);
  340. this[job.name]
  341. .apply(job, [job.payload])
  342. .then(response => {
  343. // if (!options.isQuiet)
  344. this.log("INFO", `Ran job ${job.name} (${job.toString()}) successfully`);
  345. job.setStatus("FINISHED");
  346. this.jobStatistics[job.name].successful += 1;
  347. if (
  348. config.debug &&
  349. config.debug.stationIssue === true &&
  350. config.debug.captureJobs &&
  351. config.debug.captureJobs.indexOf(job.name) !== -1
  352. ) {
  353. this.moduleManager.debugJobs.completed.push({
  354. status: "success",
  355. job,
  356. response
  357. });
  358. }
  359. job.onFinish.resolve(response);
  360. })
  361. .catch(error => {
  362. this.log("INFO", `Running job ${job.name} (${job.toString()}) failed`);
  363. job.setStatus("FINISHED");
  364. this.jobStatistics[job.name].failed += 1;
  365. if (
  366. config.debug &&
  367. config.debug.stationIssue === true &&
  368. config.debug.captureJobs &&
  369. config.debug.captureJobs.indexOf(job.name) !== -1
  370. ) {
  371. this.moduleManager.debugJobs.completed.push({
  372. status: "error",
  373. job,
  374. error
  375. });
  376. }
  377. job.onFinish.reject(error);
  378. })
  379. .finally(() => {
  380. const endTime = Date.now();
  381. const executionTime = endTime - startTime;
  382. this.jobStatistics[job.name].total += 1;
  383. this.jobStatistics[job.name].averageTiming.update(executionTime);
  384. this.runningJobs.splice(this.runningJobs.indexOf(job), 1);
  385. resolve();
  386. });
  387. });
  388. }
  389. }