From dd4f76fe6056704210567285930840082af8810a Mon Sep 17 00:00:00 2001 From: fatchan Date: Sat, 27 Apr 2019 19:19:22 +0000 Subject: [PATCH] max threads pruning and some of the batch deletion changes references #23 --- controllers/forms.js | 7 ++- db/posts.js | 83 ++++++++++++++++++-------------- helpers/files/deletepostfiles.js | 19 ++++++++ models/forms/delete-post.js | 56 +++++++++++---------- views/mixins/post.pug | 2 + 5 files changed, 101 insertions(+), 66 deletions(-) create mode 100644 helpers/files/deletepostfiles.js diff --git a/controllers/forms.js b/controllers/forms.js index a20546f1..e5861569 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -383,7 +383,7 @@ router.post('/board/:board/actions', Boards.exists, banCheck, paramConverter, ve } } if (req.body.delete) { - const { message } = await deletePosts(req, res, next, passwordPosts); + const { message } = await deletePosts(req, res, next, passwordPosts, req.params.board); messages.push(message); aggregateNeeded = true; } else { @@ -590,7 +590,10 @@ router.post('/global/actions', checkPermsMiddleware, paramConverter, async(req, let aggregateNeeded = false; try { if (req.body.global_ban) { - const { message } = await banPoster(req, res, next, null, posts); + const { message, action, query } = await banPoster(req, res, next, null, posts); + if (action) { + combinedQuery[action] = { ...combinedQuery[action], ...query} + } messages.push(message); } if (req.body.delete) { diff --git a/db/posts.js b/db/posts.js index f97f87a3..1d027c32 100644 --- a/db/posts.js +++ b/db/posts.js @@ -2,6 +2,7 @@ const Mongo = require(__dirname+'/db.js') , Boards = require(__dirname+'/boards.js') + , deletePostFiles = require(__dirname+'/../helpers/files/deletepostfiles.js') , db = Mongo.client.db('jschan').collection('posts'); module.exports = { @@ -129,7 +130,6 @@ module.exports = { }, getThreadPosts: (board, id) => { - // all posts within a thread return db.find({ 'thread': id, @@ -145,7 +145,24 @@ module.exports = { }).sort({ '_id': 1 }).toArray(); + }, + getMultipleThreadPosts: (board, ids) => { + //all posts from multiple threads in a single board + return db.find({ + 'board': board, + 'thread': { + '$in': ids + } + }, { + 'projection': { + 'salt': 0 , + 'password': 0, + 'ip': 0, + 'reports': 0, + 'globalreports': 0, + } + }).toArray(); }, getCatalog: (board) => { @@ -269,7 +286,9 @@ module.exports = { const postId = await Boards.getNextId(board); data.postId = postId; await db.insertOne(data); - //await module.exports.pruneOldThreads(board); + if (!data.thread) { //if we just added a new thread, prune anyold ones + await module.exports.pruneOldThreads(board); + } return postId; }, @@ -309,49 +328,43 @@ module.exports = { return db.deleteOne(options); }, -/* pruneOldThreads: async (board) => { - const threadsToPrune = await db.find({ + pruneOldThreads: async (board) => { + //get lowest bumped threads + const threads = await db.find({ 'thread': null, 'board': board - }, { - 'projection': { - 'postId': 1, - '_id': 0 - } }).sort({ - 'sticky': -1, - 'bumped': -1 - }).skip(3).toArray(); - console.log(threadsToPrune); - const ids = threadsToPrune.map(x => x.postId); - console.log(ids); - const data = await db.deleteMany({ - 'board': board, - '$or': [ - { - 'postId': { - '$in': ids - } - }, - { - 'thread': { - '$in': ids - } - }, - ] - }); - console.log(data.deletedCount); - //wont delete files gotta fetch thread posts for that - },*/ + 'sticky': -1, + 'bumped': -1 + }).skip(100).toArray(); //100 therads in board limit for now + //if there are any + if (threads.length > 0) { + //get the postIds + const threadIds = threads.map(thread => thread.postId); + //get all the posts from those threads + const threadPosts = await module.exports.getMultipleThreadPosts(board, threadIds); + //combine them + const postsAndThreads = threads.concat(threadPosts); + //get the filenames and delete all the files + let fileNames = []; + postsAndThreads.forEach(post => { + fileNames = fileNames.concat(post.files.map(x => x.filename)) + }); + if (fileNames.length > 0) { + await deletePostFiles(fileNames); + } + //get the mongoIds and delete them all + const postMongoIds = postsAndThreads.map(post => Mongo.ObjectId(post._id)); + await module.exports.deleteMany(postMongoIds); + } + }, deleteMany: (ids) => { - return db.deleteMany({ '_id': { '$in': ids } }); - }, deleteAll: (board) => { diff --git a/helpers/files/deletepostfiles.js b/helpers/files/deletepostfiles.js new file mode 100644 index 00000000..c1a29f69 --- /dev/null +++ b/helpers/files/deletepostfiles.js @@ -0,0 +1,19 @@ +'use strict'; + +const util = require('util') + , fs = require('fs') + , unlink = util.promisify(fs.unlink) + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + +module.exports = (fileNames) => { + + //delete all the psot files and thumbs using the filenames + return Promise.all(fileNames.map(async filename => { + //dont question it. + return Promise.all([ + unlink(`${uploadDirectory}img/${filename}`), + unlink(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`) + ]).catch(e => console.error) //ignore for now + })); + +} diff --git a/models/forms/delete-post.js b/models/forms/delete-post.js index 46889c1e..586d115c 100644 --- a/models/forms/delete-post.js +++ b/models/forms/delete-post.js @@ -1,32 +1,34 @@ 'use strict'; -const path = require('path') - , util = require('util') - , fs = require('fs') - , unlink = util.promisify(fs.unlink) - , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') +const uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + , deletePostFiles = require(__dirname+'/../../helpers/files/deletepostfiles.js') , Mongo = require(__dirname+'/../../db/db.js') , Posts = require(__dirname+'/../../db/posts.js'); -module.exports = async (req, res, next, posts) => { +module.exports = async (req, res, next, posts, board) => { - //filter to threads, then get the board and thread for each - const boardThreads = posts.filter(x => x.thread == null).map(x => { - return { - board: x.board, - thread: x.postId - }; - }); + //filter to threads + const threads = posts.filter(x => x.thread == null); //get posts from all threads let threadPosts = [] - await Promise.all(boardThreads.map(async data => { - const currentThreadPosts = await Posts.getThreadPosts(data.board, data.thread); - threadPosts = threadPosts.concat(currentThreadPosts); - return; - })) - - //combine them all into one array + if (threads.length > 0) { + if (board) { + //if this is board-specific, we can use a single query + const threadPostIds = threads.map(thread => thread.postId); + threadPosts = await Posts.getMultipleThreadPosts(board, threadPostIds); + } else { + //otherwise we fetch posts from threads on different boards separarely + //TODO: combine queries from the same board, or ideally construct a large $or query so this can be tackled in a single db query + await Promise.all(threads.map(async thread => { + //for each thread, fetch all posts from the matching board and thread matching the threads postId + const currentThreadPosts = await Posts.getThreadPosts(thread.board, thread.postId); + threadPosts = threadPosts.concat(currentThreadPosts); + })); + } + } + + //combine them all into one array, there may be duplicates but it shouldnt matter const allPosts = posts.concat(threadPosts) //delete posts from DB @@ -39,16 +41,12 @@ module.exports = async (req, res, next, posts) => { fileNames = fileNames.concat(post.files.map(x => x.filename)) }) - //delete all the files using the filenames - await Promise.all(fileNames.map(async filename => { - //dont question it. - return Promise.all([ - unlink(`${uploadDirectory}img/${filename}`), - unlink(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.png`) - ]) - })); + //delete post files + if (fileNames.length > 0) { + await deletePostFiles(fileNames); + } //hooray! - return { message:`Deleted ${boardThreads.length} threads and ${deletedPosts-boardThreads.length} posts` }; + return { message:`Deleted ${threads.length} threads and ${deletedPosts-threads.length} posts` }; } diff --git a/views/mixins/post.pug b/views/mixins/post.pug index 73dff1b7..07ff5913 100644 --- a/views/mixins/post.pug +++ b/views/mixins/post.pug @@ -77,9 +77,11 @@ mixin post(post, truncate, manage=false, globalmanage=false) each report in post.reports .reports.post-container span Date: #{report.date.toLocaleString()} + | span Reason: #{report.reason} if globalmanage === true each report in post.globalreports .reports.post-container span Date: #{report.date.toLocaleString()} + | span Reason: #{report.reason}