feat: worked further on admin import jobs

backend/logic/actions/youtube.js

@@ -358,6 +358,8 @@ export default {
 	 * @param {Function} cb - gets called with the result
 	 */
 	requestSetAdmin: isAdminRequired(async function requestSetAdmin(session, url, musicOnly, returnVideos, cb) {
+		const importJobModel = await DBModule.runJob("GET_MODEL", { modelName: "importJob" }, this);
+
 		this.keepLongJob();
 		this.publishProgress({
 			status: "started",
@@ -365,162 +367,85 @@ export default {
 			message: "Importing playlist.",
 			id: this.toString()
 		});
-		await CacheModule.runJob("RPUSH", { key: `longJobs.requestSetAdmin`, value: this.toString() }, this);
-		await CacheModule.runJob(
-			"HSET",
-			{
-				table: `requestSetAdmin`,
-				key: this.toString(),
-				value: { payload: { url, musicOnly, returnVideos }, response: null }
-			},
-			this
-		);
+		await CacheModule.runJob("RPUSH", { key: `longJobs.${session.userId}`, value: this.toString() }, this);
 
-		YouTubeModule.runJob("REQUEST_SET", { url, musicOnly, returnVideos }, this)
-			.then(async response => {
-				this.log(
-					"SUCCESS",
-					"REQUEST_SET_ADMIN",
-					`Successfully imported a YouTube playlist to be requested for admin "${session.userId}".`
-				);
-				this.publishProgress({
-					status: "success",
-					message: `Playlist is done importing. ${response.successful} were added succesfully, ${response.failed} failed (${response.alreadyInDatabase} were already in database)`
-				});
-				await CacheModule.runJob(
-					"HSET",
-					{
-						table: `requestSetAdmin`,
-						key: this.toString(),
-						value: { payload: { url, musicOnly, returnVideos }, response }
-					},
-					this
-				);
-				return cb({
-					status: "success",
-					message: `Playlist is done importing. ${response.successful} were added succesfully, ${response.failed} failed (${response.alreadyInDatabase} were already in database)`,
-					videos: returnVideos ? response.videos : null
-				});
-			})
-			.catch(async err => {
-				err = await UtilsModule.runJob("GET_ERROR", { error: err }, this);
-				this.log(
-					"ERROR",
-					"REQUEST_SET_ADMIN",
-					`Importing a YouTube playlist to be requested failed for admin "${session.userId}". "${err}"`
-				);
-				this.publishProgress({
-					status: "error",
-					message: err
-				});
-				return cb({ status: "error", message: err });
-			});
-	}),
-
-	/**
-	 * Gets request set admin long jobs
-	 *
-	 * @param {object} session - the session object automatically added by the websocket
-	 * @param {Function} cb - gets called with the result
-	 */
-	getRequestSetAdminLongJobs: isLoginRequired(async function getRequestSetAdminLongJobs(session, cb) {
 		async.waterfall(
 			[
 				next => {
-					CacheModule.runJob(
-						"LRANGE",
+					importJobModel.create(
 						{
-							key: `longJobs.requestSetAdmin`
+							type: "youtube",
+							query: {
+								url,
+								musicOnly
+							},
+							status: "in-progress",
+							response: {},
+							requestedBy: session.userId,
+							requestedAt: Date.now()
 						},
-						this
-					)
-						.then(longJobUuids => next(null, longJobUuids))
-						.catch(next);
+						next
+					);
 				},
 
-				(longJobUuids, next) => {
-					next(
-						null,
-						longJobUuids.map(longJobUuid => ({
-							job: moduleManager.jobManager.getJob(longJobUuid),
-							uuid: longJobUuid
-						}))
-					);
+				(importJob, next) => {
+					YouTubeModule.runJob("REQUEST_SET", { url, musicOnly, returnVideos }, this)
+						.then(response => {
+							next(null, importJob, response);
+						})
+						.catch(err => {
+							next(err, importJob);
+						});
 				},
 
-				(longJobs, next) => {
-					async.eachLimit(
-						longJobs,
-						1,
-						(longJob, next) => {
-							CacheModule.runJob("HGET", { table: "requestSetAdmin", key: longJob.uuid }, this)
-								.then(value => {
-									if (value) {
-										const { payload, response } = value;
-										longJob.payload = payload;
-										longJob.response = response;
-									}
-									next();
-								})
-								.catch(console.log);
+				(importJob, response, next) => {
+					importJobModel.updateOne(
+						{ _id: importJob._id },
+						{
+							$set: {
+								status: "success",
+								response: {
+									failed: response.failed,
+									successful: response.successful,
+									alreadyInDatabase: response.alreadyInDatabase,
+									successfulVideoIds: response.successfulVideoIds,
+									failedVideoIds: response.failedVideoIds
+								}
+							}
 						},
 						err => {
-							next(err, longJobs);
+							next(err, importJob, response);
 						}
 					);
-				},
-
-				(longJobs, next) => {
-					longJobs.forEach(({ job }) => {
-						if (job && job.onProgress)
-							job.onProgress.on("progress", data => {
-								this.publishProgress(
-									{
-										id: job.toString(),
-										...data
-									},
-									true
-								);
-							});
-					});
-
-					next(
-						null,
-						longJobs.map(({ job, uuid, payload, response }) => ({
-							id: uuid,
-							name: job ? job.longJobTitle : "",
-							status: job ? job.lastProgressData.status : "",
-							message: job ? job.lastProgressData.message : "",
-							payload,
-							response
-						}))
-					);
 				}
 			],
-			async (err, longJobs) => {
+			async (err, importJob, response) => {
 				if (err) {
 					err = await UtilsModule.runJob("GET_ERROR", { error: err }, this);
-
 					this.log(
 						"ERROR",
-						"GET_REQUEST_SET_ADMIN_LONG_JOBS",
-						`Couldn't get request set admin long jobs for user "${session.userId}". "${err}"`
+						"REQUEST_SET_ADMIN",
+						`Importing a YouTube playlist to be requested failed for admin "${session.userId}". "${err}"`
 					);
-
+					importJobModel.updateOne({ _id: importJob._id }, { $set: { status: "error" } });
 					return cb({ status: "error", message: err });
 				}
 
 				this.log(
 					"SUCCESS",
-					"GET_REQUEST_SET_ADMIN_LONG_JOBS",
-					`Got request set admin  long jobs for user "${session.userId}".`
+					"REQUEST_SET_ADMIN",
+					`Successfully imported a YouTube playlist to be requested for admin "${session.userId}".`
 				);
 
+				this.publishProgress({
+					status: "success",
+					message: `Playlist is done importing. ${response.successful} were added succesfully, ${response.failed} failed (${response.alreadyInDatabase} were already in database)`
+				});
+
 				return cb({
 					status: "success",
-					data: {
-						longJobs
-					}
+					message: `Playlist is done importing. ${response.successful} were added succesfully, ${response.failed} failed (${response.alreadyInDatabase} were already in database)`,
+					videos: returnVideos ? response.videos : null
 				});
 			}
 		);

backend/logic/db/index.js

@@ -17,7 +17,8 @@ const REQUIRED_DOCUMENT_VERSIONS = {
 	user: 3,
 	youtubeApiRequest: 1,
 	youtubeVideo: 1,
-	ratings: 1
+	ratings: 1,
+	importJob: 1
 };
 
 const regex = {
@@ -98,6 +99,7 @@ class _DBModule extends CoreClass {
 					await importSchema("youtubeApiRequest");
 					await importSchema("youtubeVideo");
 					await importSchema("ratings");
+					await importSchema("importJob");
 
 					this.models = {
 						song: mongoose.model("song", this.schemas.song),
@@ -112,7 +114,8 @@ class _DBModule extends CoreClass {
 						punishment: mongoose.model("punishment", this.schemas.punishment),
 						youtubeApiRequest: mongoose.model("youtubeApiRequest", this.schemas.youtubeApiRequest),
 						youtubeVideo: mongoose.model("youtubeVideo", this.schemas.youtubeVideo),
-						ratings: mongoose.model("ratings", this.schemas.ratings)
+						ratings: mongoose.model("ratings", this.schemas.ratings),
+						importJob: mongoose.model("importJob", this.schemas.importJob)
 					};
 
 					mongoose.connection.on("error", err => {
@@ -257,6 +260,7 @@ class _DBModule extends CoreClass {
 					this.models.youtubeApiRequest.syncIndexes();
 					this.models.youtubeVideo.syncIndexes();
 					this.models.ratings.syncIndexes();
+					this.models.importJob.syncIndexes();
 
 					if (config.get("skipDbDocumentsVersionCheck")) resolve();
 					else {

backend/logic/db/schemas/importJob.js

@@ -0,0 +1,9 @@
+export default {
+	type: { type: String, required: true, enum: ["youtube"] },
+	query: { type: Object, required: true },
+	status: { type: String, required: true, enum: ["success", "error", "in-progress"] },
+	response: { type: Object, required: true },
+	requestedBy: { type: String, required: true },
+	requestedAt: { type: Date, default: Date.now, required: true },
+	documentVersion: { type: Number, default: 1, required: true }
+};

backend/logic/youtube.js

@@ -1651,10 +1651,14 @@ class _YouTubeModule extends CoreClass {
 
 					(youtubeIds, next) => {
 						let successful = 0;
-						let videos = {};
 						let failed = 0;
 						let alreadyInDatabase = 0;
 
+						let videos = {};
+
+						const successfulVideoIds = [];
+						const failedVideoIds = [];
+
 						if (youtubeIds.length === 0) next();
 
 						async.eachOfLimit(
@@ -1664,11 +1668,14 @@ class _YouTubeModule extends CoreClass {
 								YouTubeModule.runJob("GET_VIDEO", { identifier: youtubeId, createMissing: true }, this)
 									.then(res => {
 										successful += 1;
+										successfulVideoIds.push(youtubeId);
+
 										if (res.existing) alreadyInDatabase += 1;
 										if (res.video) videos[index] = res.video;
 									})
 									.catch(() => {
 										failed += 1;
+										failedVideoIds.push(youtubeId);
 									})
 									.finally(() => {
 										next2();
@@ -1680,7 +1687,14 @@ class _YouTubeModule extends CoreClass {
 										.sort()
 										.map(key => videos[key]);
 
-								next(null, { successful, failed, alreadyInDatabase, videos });
+								next(null, {
+									successful,
+									failed,
+									alreadyInDatabase,
+									videos,
+									successfulVideoIds,
+									failedVideoIds
+								});
 							}
 						);
 					}

frontend/src/pages/Admin/Songs/Import.vue

@@ -134,14 +134,14 @@ export default {
 		})
 	},
 	mounted() {
-		this.socket.dispatch("youtube.getRequestSetAdminLongJobs", {
-			cb: res => {
-				console.log(111, res);
-			},
-			onProgress: res => {
-				console.log(222, res);
-			}
-		});
+		// this.socket.dispatch("youtube.getRequestSetAdminLongJobs", {
+		// 	cb: res => {
+		// 		console.log(111, res);
+		// 	},
+		// 	onProgress: res => {
+		// 		console.log(222, res);
+		// 	}
+		// });
 	},
 	methods: {
 		submitCreateImport(stage) {