max threads pruning and some of the batch deletion changes references #23

controllers/forms.js

@@ -383,7 +383,7 @@ router.post('/board/:board/actions', Boards.exists, banCheck, paramConverter, ve
 			}
 		}
 		if (req.body.delete) {
-			const { message } = await deletePosts(req, res, next, passwordPosts);
+			const { message } = await deletePosts(req, res, next, passwordPosts, req.params.board);
 			messages.push(message);
 			aggregateNeeded = true;
 		} else {
@@ -590,7 +590,10 @@ router.post('/global/actions', checkPermsMiddleware, paramConverter, async(req,
 	let aggregateNeeded = false;
 	try {
 		if (req.body.global_ban) {
-			const { message } = await banPoster(req, res, next, null, posts);
+			const { message, action, query } = await banPoster(req, res, next, null, posts);
+			if (action) {
+				combinedQuery[action] = { ...combinedQuery[action], ...query}
+			}
 			messages.push(message);
 		}
 		if (req.body.delete) {

db/posts.js

@@ -2,6 +2,7 @@
 
 const Mongo = require(__dirname+'/db.js')
 	, Boards = require(__dirname+'/boards.js')
+	, deletePostFiles = require(__dirname+'/../helpers/files/deletepostfiles.js')
 	, db = Mongo.client.db('jschan').collection('posts');
 
 module.exports = {
@@ -129,7 +130,6 @@ module.exports = {
 	},
 
 	getThreadPosts: (board, id) => {
-
 		// all posts within a thread
 		return db.find({
 			'thread': id,
@@ -145,7 +145,24 @@ module.exports = {
 		}).sort({
 			'_id': 1
 		}).toArray();
+	},
 
+	getMultipleThreadPosts: (board, ids) => {
+		//all posts from multiple threads in a single board
+		return db.find({
+			'board': board,
+			'thread': {
+				'$in': ids
+			}
+		}, {
+			'projection': {
+				'salt': 0 ,
+				'password': 0,
+				'ip': 0,
+				'reports': 0,
+				'globalreports': 0,
+			}
+		}).toArray();
 	},
 
 	getCatalog: (board) => {
@@ -269,7 +286,9 @@ module.exports = {
 		const postId = await Boards.getNextId(board);
 		data.postId = postId;
 		await db.insertOne(data);
-		//await module.exports.pruneOldThreads(board);
+		if (!data.thread) { //if we just added a new thread, prune anyold ones
+			await module.exports.pruneOldThreads(board);
+		}
 		return postId;
 
 	},
@@ -309,49 +328,43 @@ module.exports = {
 		return db.deleteOne(options);
 	},
 
-/*	pruneOldThreads: async (board) => {
-		const threadsToPrune = await db.find({
+	pruneOldThreads: async (board) => {
+		//get lowest bumped threads
+		const threads = await db.find({
 			'thread': null,
 			'board': board
-		}, {
-			'projection': {
-				'postId': 1,
-				'_id': 0
-			}
 		}).sort({
-            'sticky': -1,
-            'bumped': -1
-        }).skip(3).toArray();
-		console.log(threadsToPrune);
-		const ids = threadsToPrune.map(x => x.postId);
-		console.log(ids);
-		const data = await db.deleteMany({
-			'board': board,
-			'$or': [
-				{
-					'postId': {
-						'$in': ids
-					}
-				},
-				{
-					'thread': {
-						'$in': ids
-					}
-				},
-			]
-		});
-		console.log(data.deletedCount);
-		//wont delete files gotta fetch thread posts for that
-	},*/
+			'sticky': -1,
+			'bumped': -1
+		}).skip(100).toArray(); //100 therads in board limit for now
+		//if there are any
+		if (threads.length > 0) {
+			//get the postIds
+			const threadIds = threads.map(thread => thread.postId);
+			//get all the posts from those threads
+			const threadPosts = await module.exports.getMultipleThreadPosts(board, threadIds);
+			//combine them
+			const postsAndThreads = threads.concat(threadPosts);
+			//get the filenames and delete all the files
+		 	let fileNames = [];
+		 	postsAndThreads.forEach(post => {
+				fileNames = fileNames.concat(post.files.map(x => x.filename))
+			});
+			if (fileNames.length > 0) {
+				await deletePostFiles(fileNames);
+			}
+			//get the mongoIds and delete them all
+			const postMongoIds = postsAndThreads.map(post => Mongo.ObjectId(post._id));
+			await module.exports.deleteMany(postMongoIds);
+		}
+	},
 
 	deleteMany: (ids) => {
-
 		return db.deleteMany({
 			'_id': {
 				'$in': ids
 			}
 		});
-
 	},
 
 	deleteAll: (board) => {

helpers/files/deletepostfiles.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const util = require('util')
+	, fs = require('fs')
+	, unlink = util.promisify(fs.unlink)
+	, uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js')
+
+module.exports = (fileNames) => {
+
+	//delete all the psot files and thumbs using the filenames
+	return Promise.all(fileNames.map(async filename => {
+		//dont question it.
+		return Promise.all([
+			unlink(`${uploadDirectory}img/${filename}`),
+			unlink(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`)
+		]).catch(e => console.error) //ignore for now
+	}));
+
+}

models/forms/delete-post.js

@@ -1,32 +1,34 @@
 'use strict';
 
-const path = require('path')
-	, util = require('util')
-	, fs = require('fs')
-	, unlink = util.promisify(fs.unlink)
-	, uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js')
+const uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js')
+	, deletePostFiles = require(__dirname+'/../../helpers/files/deletepostfiles.js')
 	, Mongo = require(__dirname+'/../../db/db.js')
 	, Posts = require(__dirname+'/../../db/posts.js');
 
-module.exports = async (req, res, next, posts) => {
+module.exports = async (req, res, next, posts, board) => {
 
-	//filter to threads, then get the board and thread for each 
-	const boardThreads = posts.filter(x => x.thread == null).map(x => {
-		return {
-			board: x.board,
-			thread: x.postId
-		};
-	});
+	//filter to threads
+	const threads = posts.filter(x => x.thread == null);
 
 	//get posts from all threads
 	let threadPosts = []
-	await Promise.all(boardThreads.map(async data => {
-		const currentThreadPosts = await Posts.getThreadPosts(data.board, data.thread);
-		threadPosts = threadPosts.concat(currentThreadPosts);
-		return;
-	}))
-
-	//combine them all into one array
+	if (threads.length > 0) {
+		if (board) {
+			//if this is board-specific, we can use a single query
+			const threadPostIds = threads.map(thread => thread.postId);
+			threadPosts = await Posts.getMultipleThreadPosts(board, threadPostIds);
+		} else {
+			//otherwise we fetch posts from threads on different boards separarely
+			//TODO: combine queries from the same board, or ideally construct a large $or query so this can be tackled in a single db query
+			await Promise.all(threads.map(async thread => {
+				//for each thread, fetch all posts from the matching board and thread matching the threads postId
+				const currentThreadPosts = await Posts.getThreadPosts(thread.board, thread.postId);
+				threadPosts = threadPosts.concat(currentThreadPosts);
+			}));
+		}
+	}
+
+	//combine them all into one array, there may be duplicates but it shouldnt matter
 	const allPosts = posts.concat(threadPosts)
 
 	//delete posts from DB
@@ -39,16 +41,12 @@ module.exports = async (req, res, next, posts) => {
 		fileNames = fileNames.concat(post.files.map(x => x.filename))
 	})
 
-	//delete all the files using the filenames
-	await Promise.all(fileNames.map(async filename => {
-		//dont question it.
-		return Promise.all([
-			unlink(`${uploadDirectory}img/${filename}`),
-			unlink(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.png`)
-		])
-	}));
+	//delete post files
+	if (fileNames.length > 0) {
+		await deletePostFiles(fileNames);
+	}
 
 	//hooray!
-	return { message:`Deleted ${boardThreads.length} threads and ${deletedPosts-boardThreads.length} posts` };
+	return { message:`Deleted ${threads.length} threads and ${deletedPosts-threads.length} posts` };
 
 }

views/mixins/post.pug

@@ -77,9 +77,11 @@ mixin post(post, truncate, manage=false, globalmanage=false)
 		each report in post.reports
 			.reports.post-container
 				span Date: #{report.date.toLocaleString()}
+				| 
 				span Reason: #{report.reason}
 	if globalmanage === true
 		each report in post.globalreports
 			.reports.post-container
 				span Date: #{report.date.toLocaleString()}
+				| 
 				span Reason: #{report.reason}