wikidata.js 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. import config from "config";
  2. import axios from "axios";
  3. import CoreClass from "../core";
  4. class RateLimitter {
  5. /**
  6. * Constructor
  7. *
  8. * @param {number} timeBetween - The time between each allowed WikiData request
  9. */
  10. constructor(timeBetween) {
  11. this.dateStarted = Date.now();
  12. this.timeBetween = timeBetween;
  13. }
  14. /**
  15. * Returns a promise that resolves whenever the ratelimit of a WikiData request is done
  16. *
  17. * @returns {Promise} - promise that gets resolved when the rate limit allows it
  18. */
  19. continue() {
  20. return new Promise(resolve => {
  21. if (Date.now() - this.dateStarted >= this.timeBetween) resolve();
  22. else setTimeout(resolve, this.dateStarted + this.timeBetween - Date.now());
  23. });
  24. }
  25. /**
  26. * Restart the rate limit timer
  27. */
  28. restart() {
  29. this.dateStarted = Date.now();
  30. }
  31. }
  32. let WikiDataModule;
  33. let CacheModule;
  34. let DBModule;
  35. let MediaModule;
  36. let SongsModule;
  37. let StationsModule;
  38. let PlaylistsModule;
  39. let WSModule;
  40. class _WikiDataModule extends CoreClass {
  41. // eslint-disable-next-line require-jsdoc
  42. constructor() {
  43. super("wikidata", {
  44. concurrency: 10
  45. // priorities: {
  46. // GET_PLAYLIST: 11
  47. // }
  48. });
  49. WikiDataModule = this;
  50. }
  51. /**
  52. * Initialises the activities module
  53. *
  54. * @returns {Promise} - returns promise (reject, resolve)
  55. */
  56. async initialize() {
  57. CacheModule = this.moduleManager.modules.cache;
  58. DBModule = this.moduleManager.modules.db;
  59. MediaModule = this.moduleManager.modules.media;
  60. SongsModule = this.moduleManager.modules.songs;
  61. StationsModule = this.moduleManager.modules.stations;
  62. PlaylistsModule = this.moduleManager.modules.playlists;
  63. WSModule = this.moduleManager.modules.ws;
  64. this.genericApiRequestModel = this.GenericApiRequestModel = await DBModule.runJob("GET_MODEL", {
  65. modelName: "genericApiRequest"
  66. });
  67. // this.youtubeVideoModel = this.YoutubeVideoModel = await DBModule.runJob("GET_MODEL", {
  68. // modelName: "youtubeVideo"
  69. // });
  70. // return new Promise(resolve => {
  71. // CacheModule.runJob("SUB", {
  72. // channel: "youtube.removeYoutubeApiRequest",
  73. // cb: requestId => {
  74. // WSModule.runJob("EMIT_TO_ROOM", {
  75. // room: `view-api-request.${requestId}`,
  76. // args: ["event:youtubeApiRequest.removed"]
  77. // });
  78. // WSModule.runJob("EMIT_TO_ROOM", {
  79. // room: "admin.youtube",
  80. // args: ["event:admin.youtubeApiRequest.removed", { data: { requestId } }]
  81. // });
  82. // }
  83. // });
  84. // CacheModule.runJob("SUB", {
  85. // channel: "youtube.removeVideos",
  86. // cb: videoIds => {
  87. // const videos = Array.isArray(videoIds) ? videoIds : [videoIds];
  88. // videos.forEach(videoId => {
  89. // WSModule.runJob("EMIT_TO_ROOM", {
  90. // room: `view-youtube-video.${videoId}`,
  91. // args: ["event:youtubeVideo.removed"]
  92. // });
  93. // WSModule.runJob("EMIT_TO_ROOM", {
  94. // room: "admin.youtubeVideos",
  95. // args: ["event:admin.youtubeVideo.removed", { data: { videoId } }]
  96. // });
  97. // WSModule.runJob("EMIT_TO_ROOMS", {
  98. // rooms: ["import-album", "edit-songs"],
  99. // args: ["event:admin.youtubeVideo.removed", { videoId }]
  100. // });
  101. // });
  102. // }
  103. // });
  104. this.rateLimiter = new RateLimitter(1100);
  105. // this.requestTimeout = config.get("apis.youtube.requestTimeout");
  106. this.requestTimeout = 5000;
  107. this.axios = axios.create();
  108. // this.axios.defaults.raxConfig = {
  109. // instance: this.axios,
  110. // retry: config.get("apis.youtube.retryAmount"),
  111. // noResponseRetries: config.get("apis.youtube.retryAmount")
  112. // };
  113. // rax.attach(this.axios);
  114. // this.youtubeApiRequestModel
  115. // .find(
  116. // { date: { $gte: new Date() - 2 * 24 * 60 * 60 * 1000 } },
  117. // { date: true, quotaCost: true, _id: false }
  118. // )
  119. // .sort({ date: 1 })
  120. // .exec((err, youtubeApiRequests) => {
  121. // if (err) console.log("Couldn't load YouTube API requests.");
  122. // else {
  123. // this.apiCalls = youtubeApiRequests;
  124. // resolve();
  125. // }
  126. // });
  127. // resolve();
  128. // });
  129. }
  130. /**
  131. * Get WikiData data from work id
  132. *
  133. * @param {object} payload - object that contains the payload
  134. * @param {object} payload.workId - work id
  135. * @returns {Promise} - returns promise (reject, resolve)
  136. */
  137. async API_GET_DATA_FROM_MUSICBRAINZ_WORK(payload) {
  138. const { workId } = payload;
  139. const sparqlQuery = `SELECT DISTINCT ?item ?itemLabel ?YouTube_video_ID WHERE {
  140. SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE]". }
  141. {
  142. SELECT DISTINCT ?item WHERE {
  143. ?item p:P435 ?statement0.
  144. ?statement0 ps:P435 "${workId}".
  145. }
  146. LIMIT 100
  147. }
  148. OPTIONAL { ?item wdt:P1651 ?YouTube_video_ID. }
  149. }`;
  150. return WikiDataModule.runJob(
  151. "API_CALL",
  152. {
  153. url: "https://query.wikidata.org/sparql",
  154. params: {
  155. query: sparqlQuery
  156. }
  157. },
  158. this
  159. );
  160. }
  161. /**
  162. * Perform WikiData API call
  163. *
  164. * @param {object} payload - object that contains the payload
  165. * @param {object} payload.url - request url
  166. * @param {object} payload.params - request parameters
  167. * @param {object} payload.quotaCost - request quotaCost
  168. * @returns {Promise} - returns promise (reject, resolve)
  169. */
  170. async API_CALL(payload) {
  171. const { url, params } = payload;
  172. let genericApiRequest = await WikiDataModule.GenericApiRequestModel.findOne({
  173. url,
  174. params
  175. });
  176. if (genericApiRequest) return genericApiRequest._doc.responseData;
  177. await WikiDataModule.rateLimiter.continue();
  178. WikiDataModule.rateLimiter.restart();
  179. const { data: responseData } = await WikiDataModule.axios.get(url, {
  180. params,
  181. headers: {
  182. Accept: "application/sparql-results+json"
  183. },
  184. timeout: WikiDataModule.requestTimeout
  185. });
  186. if (responseData.error) throw new Error(responseData.error);
  187. genericApiRequest = new WikiDataModule.GenericApiRequestModel({
  188. url,
  189. params,
  190. responseData,
  191. date: Date.now()
  192. });
  193. genericApiRequest.save();
  194. return responseData;
  195. }
  196. }
  197. export default new _WikiDataModule();