From a818a25e917031256c910938fe7fafa0eb73ac65 Mon Sep 17 00:00:00 2001 From: fatchan Date: Wed, 8 May 2019 13:19:53 +0000 Subject: [PATCH 01/27] generate and save html to disk. actions that would cause a page to change delete the html. on the next visit, nginx will try_files, else pass to the backend which will generate the page again. CURRENTLY DOES NOT SUPPORT POST ACTIONS e.g. deletes, spoiler, sticky, etc will not cause pages to be deleted for future rebuilding. thats coming in next commits. consider this the start of actual smart building strategy to prevent templating and db hits unnecessarily. where its possible to serve a plain html page, we will do so. --- controllers/forms.js | 43 +++++++++++----------- controllers/pages.js | 20 +++++------ db/boards.js | 2 +- db/posts.js | 49 ++++++++++++++++---------- gulp/res/css/style.css | 6 ++-- helpers/captchaverify.js | 6 ++-- helpers/files/deletefailed.js | 7 ++-- helpers/files/deletepostfiles.js | 8 ++--- helpers/files/file-check-mime-types.js | 4 +-- helpers/isloggedin.js | 2 +- helpers/markdown.js | 8 ++--- helpers/quotes.js | 8 ++--- helpers/writepagehtml.js | 10 +++--- models/forms/actionhandler.js | 8 ++--- models/forms/changepassword.js | 20 +++++------ models/forms/deletebanners.js | 10 ++---- models/forms/deletepostsfiles.js | 9 ++--- models/forms/login.js | 7 ++-- models/forms/make-post.js | 41 ++++++++++++++++++--- models/forms/register.js | 4 +-- models/forms/uploadbanners.js | 2 +- models/pages/board.js | 18 ++++++---- models/pages/catalog.js | 15 +++++--- models/pages/changepassword.js | 14 ++++++-- models/pages/home.js | 7 ++-- models/pages/login.js | 17 +++++---- models/pages/manage.js | 3 +- models/pages/register.js | 16 ++++++--- models/pages/thread.js | 35 ++++++++++-------- package-lock.json | 26 ++++++++++++-- package.json | 3 +- server.js | 2 +- views/includes/boardheader.pug | 2 +- views/includes/footer.pug | 2 +- views/includes/navbar.pug | 7 ++-- views/includes/pages.pug | 15 +++++--- views/includes/postform.pug | 13 +++---- views/mixins/catalogtile.pug | 4 +-- views/mixins/post.pug | 2 +- views/pages/board.pug | 4 +-- views/pages/catalog.pug | 4 +-- views/pages/login.pug | 5 ++- views/pages/register.pug | 2 +- views/pages/thread.pug | 8 ++--- wipe.js | 22 ++++++++++-- 45 files changed, 315 insertions(+), 205 deletions(-) diff --git a/controllers/forms.js b/controllers/forms.js index d95039ad..a7e4bd4f 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -17,16 +17,17 @@ const express = require('express') , changePassword = require(__dirname+'/../models/forms/changepassword.js') , registerAccount = require(__dirname+'/../models/forms/register.js') , checkPermsMiddleware = require(__dirname+'/../helpers/haspermsmiddleware.js') - , checkPerms = require(__dirname+'/../helpers/hasperms.js') , paramConverter = require(__dirname+'/../helpers/paramconverter.js') , banCheck = require(__dirname+'/../helpers/bancheck.js') , deletePostFiles = require(__dirname+'/../helpers/files/deletepostfiles.js') , verifyCaptcha = require(__dirname+'/../helpers/captchaverify.js') , actionHandler = require(__dirname+'/../models/forms/actionhandler.js') - , csrf = require(__dirname+'/../helpers/csrfmiddleware.js'); + , csrf = require(__dirname+'/../helpers/csrfmiddleware.js') + , actionChecker = require(__dirname+'/../helpers/actionchecker.js'); + // login to account -router.post('/login', csrf, (req, res, next) => { +router.post('/login', (req, res, next) => { const errors = []; @@ -50,7 +51,7 @@ router.post('/login', csrf, (req, res, next) => { return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': '/login' + 'redirect': '/login.html' }) } @@ -98,7 +99,7 @@ router.post('/changepassword', verifyCaptcha, async (req, res, next) => { return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': '/changepassword' + 'redirect': '/changepassword.html' }) } @@ -144,7 +145,7 @@ router.post('/register', verifyCaptcha, (req, res, next) => { return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': '/register' + 'redirect': '/register.html' }) } @@ -153,7 +154,7 @@ router.post('/register', verifyCaptcha, (req, res, next) => { }); // make new post -router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, async (req, res, next) => { +router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verifyCaptcha, async (req, res, next) => { let numFiles = 0; if (req.files && req.files.file) { @@ -194,7 +195,7 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, async return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/${req.params.board}${req.body.thread ? '/thread/' + req.body.thread : ''}` + 'redirect': `/${req.params.board}${req.body.thread ? '/thread/' + req.body.thread + '.html' : ''}` }) } @@ -232,13 +233,13 @@ router.post('/board/:board/settings', csrf, Boards.exists, checkPermsMiddleware, return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }) } return res.status(501).render('message', { 'title': 'Not implemented', - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }) }); @@ -266,7 +267,7 @@ router.post('/board/:board/addbanners', csrf, Boards.exists, checkPermsMiddlewar return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }) } @@ -292,7 +293,7 @@ router.post('/board/:board/deletebanners', csrf, Boards.exists, checkPermsMiddle return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }) } @@ -301,7 +302,7 @@ router.post('/board/:board/deletebanners', csrf, Boards.exists, checkPermsMiddle return res.status(400).render('message', { 'title': 'Bad request', 'message': 'Invalid banners selected', - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }) } } @@ -333,7 +334,7 @@ router.post('/board/:board/unban', csrf, Boards.exists, checkPermsMiddleware, pa return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }); } @@ -347,7 +348,7 @@ router.post('/board/:board/unban', csrf, Boards.exists, checkPermsMiddleware, pa return res.render('message', { 'title': 'Success', 'messages': messages, - 'redirect': `/${req.params.board}/manage` + 'redirect': `/${req.params.board}/manage.html` }); }); @@ -387,7 +388,7 @@ router.post('/global/actions', csrf, checkPermsMiddleware, paramConverter, async return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': '/globalmanage' + 'redirect': '/globalmanage.html' }) } @@ -397,7 +398,7 @@ router.post('/global/actions', csrf, checkPermsMiddleware, paramConverter, async return res.status(404).render('message', { 'title': 'Not found', 'errors': 'Selected posts not found', - 'redirect': '/globalmanage' + 'redirect': '/globalmanage.html' }) } @@ -414,7 +415,7 @@ router.post('/global/actions', csrf, checkPermsMiddleware, paramConverter, async } messages.push(message); } - if (hasPerms && req.body.delete_ip_global) { + if (req.body.delete_ip_global) { const deletePostIps = posts.map(x => x.ip); const deleteIpPosts = await Posts.db.find({ 'ip': { @@ -476,7 +477,7 @@ router.post('/global/actions', csrf, checkPermsMiddleware, paramConverter, async return res.render('message', { 'title': 'Success', 'messages': messages, - 'redirect': '/globalmanage' + 'redirect': '/globalmanage.html' }); }); @@ -493,7 +494,7 @@ router.post('/global/unban', csrf, checkPermsMiddleware, paramConverter, async(r return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/globalmanage` + 'redirect': `/globalmanage.html` }); } @@ -507,7 +508,7 @@ router.post('/global/unban', csrf, checkPermsMiddleware, paramConverter, async(r return res.render('message', { 'title': 'Success', 'messages': messages, - 'redirect': `/globalmanage` + 'redirect': `/globalmanage.html` }); }); diff --git a/controllers/pages.js b/controllers/pages.js index d3d021de..5968e4c3 100644 --- a/controllers/pages.js +++ b/controllers/pages.js @@ -21,19 +21,19 @@ const express = require('express') , thread = require(__dirname+'/../models/pages/thread.js'); //homepage with board list -router.get('/index', home); +router.get('/index.html', home); //login page -router.get('/login', csrf, login); +router.get('/login.html', login); //registration page -router.get('/register', register); +router.get('/register.html', register); //change password page -router.get('/changepassword', changePassword); +router.get('/changepassword.html', changePassword); //logout -router.get('/logout', csrf, isLoggedIn, (req, res, next) => { +router.get('/logout', isLoggedIn, (req, res, next) => { //remove session req.session.destroy(); @@ -48,19 +48,19 @@ router.get('/captcha', captcha); router.get('/banners', banners); //board manage page -router.get('/:board/manage', Boards.exists, isLoggedIn, hasPerms, csrf, manage); +router.get('/:board/manage.html', Boards.exists, isLoggedIn, hasPerms, csrf, manage); //board manage page -router.get('/globalmanage', isLoggedIn, hasPerms, csrf, globalManage); +router.get('/globalmanage.html', isLoggedIn, hasPerms, csrf, globalManage); // board page/recents -router.get('/:board/(:page([2-9]*|index))?', Boards.exists, paramConverter, board); +router.get('/:board/:page([2-9]*|index).html', Boards.exists, paramConverter, board); // thread view page -router.get('/:board/thread/:id(\\d+)', Boards.exists, paramConverter, thread); +router.get('/:board/thread/:id(\\d+).html', Boards.exists, paramConverter, thread); // board catalog page -router.get('/:board/catalog', Boards.exists, catalog); +router.get('/:board/catalog.html', Boards.exists, catalog); module.exports = router; diff --git a/db/boards.js b/db/boards.js index 4157a550..0e225f46 100644 --- a/db/boards.js +++ b/db/boards.js @@ -78,7 +78,7 @@ module.exports = { return res.status(403).render('message', { 'title': 'Forbidden', 'message': 'You do not have permission to manage this board', - 'redirect': '/login' + 'redirect': '/login.html' }); }, diff --git a/db/posts.js b/db/posts.js index c12f2493..fa8aba1b 100644 --- a/db/posts.js +++ b/db/posts.js @@ -9,6 +9,16 @@ module.exports = { db, + getBeforeCount: (board, thread) => { + return db.countDocuments({ + 'board': board, + 'thread': null, + 'bumped': { + '$gt': thread.bumped + } + }); + }, + getRecent: async (board, page) => { // get all thread posts (posts with null thread id) const threads = await db.find({ @@ -331,27 +341,28 @@ module.exports = { }).sort({ 'sticky': -1, 'bumped': -1 - }).skip(threadLimit).toArray(); //100 therads in board limit for now + }).skip(threadLimit).toArray(); //if there are any - if (threads.length > 0) { - //get the postIds - const threadIds = threads.map(thread => thread.postId); - //get all the posts from those threads - const threadPosts = await module.exports.getMultipleThreadPosts(board, threadIds); - //combine them - const postsAndThreads = threads.concat(threadPosts); - //get the filenames and delete all the files - let fileNames = []; - postsAndThreads.forEach(post => { - fileNames = fileNames.concat(post.files.map(x => x.filename)) - }); - if (fileNames.length > 0) { - await deletePostFiles(fileNames); - } - //get the mongoIds and delete them all - const postMongoIds = postsAndThreads.map(post => Mongo.ObjectId(post._id)); - await module.exports.deleteMany(postMongoIds); + if (threads.length === 0) { + return; + } + //get the postIds + const threadIds = threads.map(thread => thread.postId); + //get all the posts from those threads + const threadPosts = await module.exports.getMultipleThreadPosts(board, threadIds); + //combine them + const postsAndThreads = threads.concat(threadPosts); + //get the filenames and delete all the files + let fileNames = []; + postsAndThreads.forEach(post => { + fileNames = fileNames.concat(post.files.map(x => x.filename)) + }); + if (fileNames.length > 0) { + await deletePostFiles(fileNames); } + //get the mongoIds and delete them all + const postMongoIds = postsAndThreads.map(post => Mongo.ObjectId(post._id)); + return module.exports.deleteMany(postMongoIds); }, deleteMany: (ids) => { diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index 35cc1445..115e36ef 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -156,8 +156,8 @@ object { font-weight: bold; } -.redtext { - color: maroon; +.pinktext { + color: #E0727F; } .greentext { @@ -254,7 +254,7 @@ td, th { display: flex; flex-direction: column; max-width: 100%; - /*margin-top: 10px;*/ + width: 400px; } .togglable { diff --git a/helpers/captchaverify.js b/helpers/captchaverify.js index c681667b..8aca1508 100644 --- a/helpers/captchaverify.js +++ b/helpers/captchaverify.js @@ -2,9 +2,7 @@ const Captchas = require(__dirname+'/../db/captchas.js') , Mongo = require(__dirname+'/../db/db.js') - , util = require('util') - , fs = require('fs') - , unlink = util.promisify(fs.unlink) + , remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { @@ -46,7 +44,7 @@ module.exports = async (req, res, next) => { //it was correct, so delete the file, the cookie and continue res.clearCookie('captchaid'); - await unlink(`${uploadDirectory}captcha/${captchaId}.jpg`) + await remove(`${uploadDirectory}captcha/${captchaId}.jpg`) return next(); diff --git a/helpers/files/deletefailed.js b/helpers/files/deletefailed.js index 969e5142..d05c5ed6 100644 --- a/helpers/files/deletefailed.js +++ b/helpers/files/deletefailed.js @@ -1,15 +1,12 @@ 'use strict'; -const path = require('path') - , util = require('util') - , fs = require('fs') - , unlink = util.promisify(fs.unlink) +const remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (filenames, folder) => { await Promise.all(filenames.map(async filename => { - unlink(`${uploadDirectory}${folder}/${filename}`) + remove(`${uploadDirectory}${folder}/${filename}`) })); } diff --git a/helpers/files/deletepostfiles.js b/helpers/files/deletepostfiles.js index c1a29f69..b339e6b2 100644 --- a/helpers/files/deletepostfiles.js +++ b/helpers/files/deletepostfiles.js @@ -1,8 +1,6 @@ 'use strict'; -const util = require('util') - , fs = require('fs') - , unlink = util.promisify(fs.unlink) +const remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') module.exports = (fileNames) => { @@ -11,8 +9,8 @@ module.exports = (fileNames) => { return Promise.all(fileNames.map(async filename => { //dont question it. return Promise.all([ - unlink(`${uploadDirectory}img/${filename}`), - unlink(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`) + remove(`${uploadDirectory}img/${filename}`), + remove(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`) ]).catch(e => console.error) //ignore for now })); diff --git a/helpers/files/file-check-mime-types.js b/helpers/files/file-check-mime-types.js index 846b02b5..c6b438f9 100644 --- a/helpers/files/file-check-mime-types.js +++ b/helpers/files/file-check-mime-types.js @@ -4,12 +4,12 @@ const imageMimeTypes = new Set([ 'image/jpeg', 'image/pjpeg', 'image/png', + 'image/bmp', 'image/gif', + 'image/webp', ]); const videoMimeTypes = new Set([ - 'image/webp', - 'image/bmp', 'video/mp4', 'video/webm', ]); diff --git a/helpers/isloggedin.js b/helpers/isloggedin.js index 69d540c1..458c9ff5 100644 --- a/helpers/isloggedin.js +++ b/helpers/isloggedin.js @@ -4,5 +4,5 @@ module.exports = (req, res, next) => { if (req.session.authenticated === true) { return next(); } - res.redirect('/login'); + res.redirect('/login.html'); } diff --git a/helpers/markdown.js b/helpers/markdown.js index 9164a733..a882cf6c 100644 --- a/helpers/markdown.js +++ b/helpers/markdown.js @@ -2,7 +2,7 @@ const Posts = require(__dirname+'/../db/posts.js') , greentextRegex = /^>([^>].+)/gm - , redtextRegex = /^<([^<].+)/gm + , pinktextRegex = /^<([^<].+)/gm , boldRegex = /""(.+)""/gm , titleRegex = /==(.+)==/gm , italicRegex = /__(.+)__/gm @@ -13,9 +13,9 @@ const Posts = require(__dirname+'/../db/posts.js') module.exports = (board, thread, text) => { - //redtext - text = text.replace(redtextRegex, (match, redtext) => { - return `<${redtext}`; + //pinktext + text = text.replace(pinktextRegex, (match, pinktext) => { + return `<${pinktext}`; }); //greentext diff --git a/helpers/quotes.js b/helpers/quotes.js index 4068a8f9..46fe8d46 100644 --- a/helpers/quotes.js +++ b/helpers/quotes.js @@ -75,7 +75,7 @@ module.exports = async (board, text) => { text = text.replace(quoteRegex, (match) => { const quotenum = +match.substring(2); if (postThreadIdMap[board] && postThreadIdMap[board][quotenum]) { - return `>>${quotenum}`; + return `>>${quotenum}`; } return match; }); @@ -86,9 +86,9 @@ module.exports = async (board, text) => { const quoteboard = quote[1]; const quotenum = +quote[2]; if (postThreadIdMap[quoteboard] && postThreadIdMap[quoteboard][quotenum]) { - return `>>>/${quoteboard}/${quotenum}`; - } else if (postThreadIdMap[quoteboard] && quotenum === 0) { - return `>>>/${quoteboard}/`; + return `>>>/${quoteboard}/${quotenum}`; + } else if (!quote[2]) { + return `>>>/${quoteboard}/`; } return match; }); diff --git a/helpers/writepagehtml.js b/helpers/writepagehtml.js index 9fe0433e..3880c931 100644 --- a/helpers/writepagehtml.js +++ b/helpers/writepagehtml.js @@ -1,14 +1,12 @@ 'use strict'; -const util = require('util') - , fs = require('fs') +const outputFile = require('fs-extra').outputFile , pug = require('pug') , path = require('path') - , writeFile = util.promisify(fs.writeFile) , uploadDirectory = require(__dirname+'/uploadDirectory.js') - , pugDirectory = path.join(__dirname+'/../views/pages'); + , pugDirectory = path.join(__dirname+'/../views/pages/'); module.exports = async (htmlName, pugName, pugVars) => { - const html = pug.renderFile(`${pugDirectory}/${pugName}`, pugVars); - return writeFile(`${uploadDirectory}html/${htmlName}`, html); + const html = pug.renderFile(`${pugDirectory}${pugName}`, pugVars); + return outputFile(`${uploadDirectory}html/${htmlName}`, html); }; diff --git a/models/forms/actionhandler.js b/models/forms/actionhandler.js index 95fc013a..572d35a8 100644 --- a/models/forms/actionhandler.js +++ b/models/forms/actionhandler.js @@ -57,7 +57,7 @@ module.exports = async (req, res, next) => { return res.status(400).render('message', { 'title': 'Bad request', 'errors': errors, - 'redirect': `/${req.params.board}` + 'redirect': `/${req.params.board}/` }) } @@ -66,7 +66,7 @@ module.exports = async (req, res, next) => { return res.status(404).render('message', { 'title': 'Not found', 'error': 'Selected posts not found', - 'redirect': `/${req.params.board}` + 'redirect': `/${req.params.board}/` }) } @@ -88,7 +88,7 @@ module.exports = async (req, res, next) => { return res.status(403).render('message', { 'title': 'Forbidden', 'error': 'Password did not match any selected posts', - 'redirect': `/${req.params.board}` + 'redirect': `/${req.params.board}/` }); } } else { @@ -247,7 +247,7 @@ module.exports = async (req, res, next) => { return res.render('message', { 'title': 'Success', 'messages': messages, - 'redirect': `/${req.params.board}` + 'redirect': `/${req.params.board}/` }); } diff --git a/models/forms/changepassword.js b/models/forms/changepassword.js index 6fb034f0..c7dc6960 100644 --- a/models/forms/changepassword.js +++ b/models/forms/changepassword.js @@ -17,7 +17,7 @@ module.exports = async (req, res, next) => { return res.status(403).render('message', { 'title': 'Forbidden', 'message': 'Incorrect username or password', - 'redirect': redirect ? `/login?redirect=${redirect}` : '/changepassword' + 'redirect': '/changepassword.html' }); } @@ -25,16 +25,16 @@ module.exports = async (req, res, next) => { const passwordMatch = await bcrypt.compare(password, account.passwordHash); //if hashes matched - if (passwordMatch === true) { - //change the password - await Accounts.changePassword(username, newPassword); - return res.redirect('/login'); + if (passwordMatch === false) { + return res.status(403).render('message', { + 'title': 'Forbidden', + 'message': 'Incorrect username or password', + 'redirect': '/changepassword.html' + }); } - return res.status(403).render('message', { - 'title': 'Forbidden', - 'message': 'Incorrect username or password', - 'redirect': redirect ? `/login?redirect=${redirect}` : '/login' - }); + //change the password + await Accounts.changePassword(username, newPassword); + return res.redirect('/login.html'); } diff --git a/models/forms/deletebanners.js b/models/forms/deletebanners.js index c79f0581..f8015408 100644 --- a/models/forms/deletebanners.js +++ b/models/forms/deletebanners.js @@ -1,19 +1,15 @@ 'use strict'; -const uuidv4 = require('uuid/v4') - , path = require('path') - , util = require('util') - , fs = require('fs') - , unlink = util.promisify(fs.unlink) +const remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') , Boards = require(__dirname+'/../../db/boards.js'); module.exports = async (req, res, next) => { - const redirect = `/${req.params.board}/manage` + const redirect = `/${req.params.board}/manage.html` await Promise.all(req.body.checkedbanners.map(async filename => { - unlink(`${uploadDirectory}banner/${filename}`); + remove(`${uploadDirectory}banner/${filename}`); })); // i dont think there is a way to get the number of array items removed with $pullAll diff --git a/models/forms/deletepostsfiles.js b/models/forms/deletepostsfiles.js index 03507893..b1d80988 100644 --- a/models/forms/deletepostsfiles.js +++ b/models/forms/deletepostsfiles.js @@ -1,9 +1,6 @@ 'use strict'; -const path = require('path') - , util = require('util') - , fs = require('fs') - , unlink = util.promisify(fs.unlink) +const remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') module.exports = async (posts) => { @@ -24,8 +21,8 @@ module.exports = async (posts) => { await Promise.all(fileNames.map(async filename => { //dont question it. return Promise.all([ - unlink(`${uploadDirectory}img/${filename}`), - unlink(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.png`) + remove(`${uploadDirectory}img/${filename}`), + remove(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.png`) ]) })); diff --git a/models/forms/login.js b/models/forms/login.js index fa8e8b1a..532a6acf 100644 --- a/models/forms/login.js +++ b/models/forms/login.js @@ -7,7 +7,6 @@ module.exports = async (req, res, next) => { const username = req.body.username.toLowerCase(); const password = req.body.password; - const redirect = req.body.redirect; //fetch an account let account; @@ -22,7 +21,7 @@ module.exports = async (req, res, next) => { return res.status(403).render('message', { 'title': 'Forbidden', 'message': 'Incorrect username or password', - 'redirect': redirect ? `/login?redirect=${redirect}` : '/login' + 'redirect': '/login.html' }); } @@ -45,14 +44,14 @@ module.exports = async (req, res, next) => { req.session.authenticated = true; //successful login - return res.redirect(redirect || '/'); + return res.redirect('/'); } return res.status(403).render('message', { 'title': 'Forbidden', 'message': 'Incorrect username or password', - 'redirect': redirect ? `/login?redirect=${redirect}` : '/login' + 'redirect': '/login.html' }); } diff --git a/models/forms/make-post.js b/models/forms/make-post.js index 99463748..611c9926 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -5,6 +5,7 @@ const uuidv4 = require('uuid/v4') , util = require('util') , crypto = require('crypto') , randomBytes = util.promisify(crypto.randomBytes) + , remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') , Posts = require(__dirname+'/../../db/posts.js') , getTripCode = require(__dirname+'/../../helpers/tripcode.js') @@ -31,7 +32,7 @@ const uuidv4 = require('uuid/v4') module.exports = async (req, res, next, numFiles) => { // check if this is responding to an existing thread - let redirect = `/${req.params.board}` + let redirect = `/${req.params.board}/` let salt = null; let thread = null; const hasPerms = permsCheck(req, res); @@ -46,7 +47,7 @@ module.exports = async (req, res, next, numFiles) => { }); } salt = thread.salt; - redirect += `/thread/${req.body.thread}` + redirect += `thread/${req.body.thread}.html` if (thread.locked && !hasPerms) { return res.status(400).render('message', { 'title': 'Bad request', @@ -62,6 +63,13 @@ module.exports = async (req, res, next, numFiles) => { }); } } + if (numFiles > res.locals.board.settings.maxFiles) { + return res.status(400).render('message', { + 'title': 'Bad request', + 'message': `Too many files. Max files per post is ${res.locals.board.settings.maxFiles}.`, + 'redirect': redirect + }); + } let files = []; // if we got a file if (numFiles > 0) { @@ -214,11 +222,36 @@ module.exports = async (req, res, next, numFiles) => { } const postId = await Posts.insertOne(req.params.board, data, thread); - if (!data.thread) { //if we just added a new thread, prune any old ones + if (!data.thread) { + //if we just added a new thread, prune any old ones await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); } - const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}#${postId}`; + //now we need to delete outdated html + const removePromises = [] + //always need to refresh catalog + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/catalog.html`)); + if (data.thread) { + //refresh the thread itself + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${req.body.thread}.html`)); + if (!data.sage) { + //bumping a thread, so delete all pages above it + const numThreadsBefore = await Posts.getBeforeCount(req.params.board, thread); + const pagesToRemove = Math.ceil(numThreadsBefore/10); + for (let i = 1; i <= pagesToRemove; i++) { + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); + } + } + } else { + //new thread, remove all pages + for (let i = 1; i <= Math.ceil(res.locals.board.settings.threadLimit/10); i++) { + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); + } + } + await Promise.all(removePromises); + + const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`; return res.redirect(successRedirect); + } diff --git a/models/forms/register.js b/models/forms/register.js index 51f96bf5..f732155b 100644 --- a/models/forms/register.js +++ b/models/forms/register.js @@ -20,7 +20,7 @@ module.exports = async (req, res, next) => { return res.status(409).render('message', { 'title': 'Conflict', 'message': 'Account with this username already exists', - 'redirect': '/register' + 'redirect': '/register.html' }); } @@ -31,6 +31,6 @@ module.exports = async (req, res, next) => { return next(err); } - return res.redirect('/login') + return res.redirect('/login.html') } diff --git a/models/forms/uploadbanners.js b/models/forms/uploadbanners.js index 1d185596..b3f40416 100644 --- a/models/forms/uploadbanners.js +++ b/models/forms/uploadbanners.js @@ -11,7 +11,7 @@ const uuidv4 = require('uuid/v4') module.exports = async (req, res, next, numFiles) => { - const redirect = `/${req.params.board}/manage` + const redirect = `/${req.params.board}/manage.html` // check all mime types befoer we try saving anything for (let i = 0; i < numFiles; i++) { diff --git a/models/pages/board.js b/models/pages/board.js index efdce077..d0c021f0 100644 --- a/models/pages/board.js +++ b/models/pages/board.js @@ -1,26 +1,32 @@ 'use strict'; -const Posts = require(__dirname+'/../../db/posts.js')l +const Posts = require(__dirname+'/../../db/posts.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); module.exports = async (req, res, next) => { const page = req.params.page === 'index' ? 1 : (req.params.page || 1); let threads; let pages; + let pageURL; try { pages = Math.ceil((await Posts.getPages(req.params.board)) / 10) if (page > pages && pages > 0) { return next(); } threads = await Posts.getRecent(req.params.board, page); + pageURL = `${req.params.board}/${req.params.page}.html`; + await writePageHTML(pageURL, 'board.pug', { + board: res.locals.board, + threads: threads || [], + pages, + page + }); } catch (err) { return next(err); } - return res.render('board', { - threads: threads || [], - pages, - page, - }); + return res.sendFile(`${uploadDirectory}html/${pageURL}`); } diff --git a/models/pages/catalog.js b/models/pages/catalog.js index 2dd42f0a..189550af 100644 --- a/models/pages/catalog.js +++ b/models/pages/catalog.js @@ -1,20 +1,25 @@ 'use strict'; -const Posts = require(__dirname+'/../../db/posts.js'); +const Posts = require(__dirname+'/../../db/posts.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); module.exports = async (req, res, next) => { // get all threads let threads; + let pageURL; try { threads = await Posts.getCatalog(req.params.board); + pageURL = `${req.params.board}/catalog.html`; + await writePageHTML(pageURL, 'catalog.pug', { + board: res.locals.board, + threads: threads || [], + }); } catch (err) { return next(err); } - //render the page - res.render('catalog', { - threads: threads || [], - }); + return res.sendFile(`${uploadDirectory}html/${pageURL}`); } diff --git a/models/pages/changepassword.js b/models/pages/changepassword.js index 99f6542c..277d5329 100644 --- a/models/pages/changepassword.js +++ b/models/pages/changepassword.js @@ -1,8 +1,16 @@ 'use strict'; -module.exports = (req, res, next) => { +const writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); - //render the page - res.render('changepassword'); +module.exports = async (req, res, next) => { + + try { + await writePageHTML('changepassword.html', 'changepassword.pug'); + } catch (err) { + return next(err); + } + + return res.sendFile(`${uploadDirectory}html/changepassword.html`); } diff --git a/models/pages/home.js b/models/pages/home.js index f6f54c66..7c722622 100644 --- a/models/pages/home.js +++ b/models/pages/home.js @@ -1,6 +1,8 @@ 'use strict'; -const Boards = require(__dirname+'/../../db/boards.js'); +const Boards = require(__dirname+'/../../db/boards.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); module.exports = async (req, res, next) => { @@ -8,10 +10,11 @@ module.exports = async (req, res, next) => { let boards; try { boards = await Boards.find(); + await writePageHTML('index.html', 'home.pug', { boards }); } catch (err) { return next(err); } - res.render('home', { boards }); + return res.sendFile(`${uploadDirectory}html/index.html`); } diff --git a/models/pages/login.js b/models/pages/login.js index 373b330d..e9aadbfa 100644 --- a/models/pages/login.js +++ b/models/pages/login.js @@ -1,11 +1,16 @@ 'use strict'; -module.exports = (req, res, next) => { +const writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); - //render the page - res.render('login', { - csrf: req.csrfToken(), - redirect: req.query.redirect, - }); +module.exports = async (req, res, next) => { + + try { + await writePageHTML('login.html', 'login.pug'); + } catch (err) { + return next(err); + } + + return res.sendFile(`${uploadDirectory}html/login.html`); } diff --git a/models/pages/manage.js b/models/pages/manage.js index 962f5039..7319f3da 100644 --- a/models/pages/manage.js +++ b/models/pages/manage.js @@ -1,7 +1,8 @@ 'use strict'; const Posts = require(__dirname+'/../../db/posts.js') - , Bans = require(__dirname+'/../../db/bans.js'); + , Bans = require(__dirname+'/../../db/bans.js') + , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); module.exports = async (req, res, next) => { diff --git a/models/pages/register.js b/models/pages/register.js index f8ce78fb..58b51f1b 100644 --- a/models/pages/register.js +++ b/models/pages/register.js @@ -1,10 +1,16 @@ 'use strict'; -module.exports = (req, res, next) => { +const writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); - //render the page - res.render('register', { - csrf: req.csrfToken() - }); +module.exports = async (req, res, next) => { + + try { + await writePageHTML('register.html', 'register.pug'); + } catch (err) { + return next(err); + } + + return res.sendFile(`${uploadDirectory}html/register.html`); } diff --git a/models/pages/thread.js b/models/pages/thread.js index 2fd61447..95ad9570 100644 --- a/models/pages/thread.js +++ b/models/pages/thread.js @@ -1,23 +1,28 @@ 'use strict'; -const Posts = require(__dirname+'/../../db/posts.js'); +const Posts = require(__dirname+'/../../db/posts.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); module.exports = async (req, res, next) => { - //get the recently bumped thread & preview posts - let thread; - try { - thread = await Posts.getThread(req.params.board, req.params.id); - } catch (err) { - return next(err); - } + //get the recently bumped thread & preview posts + let thread; + let threadURL; + try { + thread = await Posts.getThread(req.params.board, req.params.id); + if (!thread) { + return res.status(404).render('404'); + } + threadURL = `${req.params.board}/thread/${req.params.id}.html`; + await writePageHTML(threadURL, 'thread.pug', { + board: res.locals.board, + thread + }); + } catch (err) { + return next(err); + } - if (!thread) { - return res.status(404).render('404'); - } + return res.sendFile(`${uploadDirectory}html/${threadURL}`); - //render the page - res.render('thread', { - thread - }); } diff --git a/package-lock.json b/package-lock.json index 045341be..ab716328 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2083,6 +2083,16 @@ "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, "fs-minipass": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", @@ -2826,8 +2836,7 @@ "graceful-fs": { "version": "4.1.15", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", - "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", - "dev": true + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" }, "gulp": { "version": "4.0.1", @@ -3572,6 +3581,14 @@ "dev": true, "optional": true }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -5920,6 +5937,11 @@ "through2-filter": "^3.0.0" } }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index 3adc6f9f..67d04b20 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "express-session": "^1.16.1", "fluent-ffmpeg": "^2.1.2", "fs": "0.0.1-security", + "fs-extra": "^7.0.1", "helmet": "^3.16.0", "mongodb": "^3.2.3", "path": "^0.12.7", @@ -33,7 +34,7 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "start": "node server.js", - "wipe": "node reset.js && gulp" + "wipe": "node wipe.js && gulp" }, "author": "", "license": "ISC" diff --git a/server.js b/server.js index 85860e4d..5fca151e 100644 --- a/server.js +++ b/server.js @@ -53,7 +53,7 @@ const express = require('express') if (req.method !== 'POST') { return next(); } - if (!req.headers.referer || !req.headers.referer.startsWith('https://fatpeople.lol')) { + if (!req.headers.referer || !req.headers.referer.match(/^https:\/\/(www\.)?fatpeople\.lol/)) { return res.status(403).render('message', { 'title': 'Forbidden', 'message': 'Invalid or missing "Referer" header. Are you posting from the correct URL?' diff --git a/views/includes/boardheader.pug b/views/includes/boardheader.pug index b49231b9..a2daad93 100644 --- a/views/includes/boardheader.pug +++ b/views/includes/boardheader.pug @@ -1,6 +1,6 @@ section.board-header if board.banners.length > 0 object.board-banner(data=`/banners?board=${board._id}` width='300' height='100') - a.no-decoration(href=`/${board._id}/index`) + a.no-decoration(href=`/${board._id}/index.html`) h1.board-title /#{board._id}/ - #{board.name} h4.board-description #{board.description} diff --git a/views/includes/footer.pug b/views/includes/footer.pug index be072f5d..25ab0523 100644 --- a/views/includes/footer.pug +++ b/views/includes/footer.pug @@ -1,2 +1,2 @@ .footer - a(href='https://github.com/fatchan/jschan/') not lynxchan ™ + a(href='https://github.com/fatchan/jschan/') not lynxchan™ diff --git a/views/includes/navbar.pug b/views/includes/navbar.pug index bd6bfef8..7e430185 100644 --- a/views/includes/navbar.pug +++ b/views/includes/navbar.pug @@ -1,8 +1,5 @@ nav.navbar a.nav-item(href='/') Home a.nav-item.right(href='/logout') Logout - if board - a.nav-item.right(href=`/login?redirect=/${board._id}/index`) Login - a.nav-item.right(href=`/${board._id}/manage`) Manage - else - a.nav-item.right(href='/login') Login + a.nav-item.right(href=`/${board ? board._id+'/' : 'global'}manage.html`) Manage + a.nav-item.right(href='/login.html') Login diff --git a/views/includes/pages.pug b/views/includes/pages.pug index d8382215..09ed2352 100644 --- a/views/includes/pages.pug +++ b/views/includes/pages.pug @@ -1,15 +1,20 @@ | Page: -span - a(href=`/${board._id}/index`) [#{1}] - | +if page === 1 + span + a(href=`/${board._id}/index.html`) [#{1}] + | +else + span + a(href=`/${board._id}/index.html`) #{1} + | - for(let i = 2; i <= pages; i++) if i === page span - a(href=`/${board._id}/${i}`) [#{i}] + a(href=`/${board._id}/${i}.html`) [#{i}] | else span - a(href=`/${board._id}/${i}`) #{i} + a(href=`/${board._id}/${i}.html`) #{i} | | | diff --git a/views/includes/postform.pug b/views/includes/postform.pug index 658d2e37..e34ef2e7 100644 --- a/views/includes/postform.pug +++ b/views/includes/postform.pug @@ -25,12 +25,13 @@ section.form-wrapper.flex-center.mv-10 section.postform-row .postform-label Message textarea#message(name='message', rows='5', autocomplete='off' maxlength='2000') - section.postform-row - .postform-label Files - input#file(type='file', name='file' multiple='multiple') - label.postform-style.ph-5.ml-1 - input#spoiler(type='checkbox', name='spoiler', value='true') - | Spoiler + if board.settings.maxFiles !== 0 + section.postform-row + .postform-label Files + input#file(type='file', name='file' multiple='multiple') + label.postform-style.ph-5.ml-1 + input#spoiler(type='checkbox', name='spoiler', value='true') + | Spoiler section.postform-row .postform-label Password input#password(type='password', name='password', autocomplete='off' placeholder='password for deleting post later' maxlength='50') diff --git a/views/mixins/catalogtile.pug b/views/mixins/catalogtile.pug index fc0ca6b1..2edd4235 100644 --- a/views/mixins/catalogtile.pug +++ b/views/mixins/catalogtile.pug @@ -1,13 +1,13 @@ mixin catalogtile(board, post, truncate) article(class='catalog-tile') - - const postURL = `/${board._id}/thread/${post.postId}#${post.postId}` + - const postURL = `/${board._id}/thread/${post.postId}.html#${post.postId}` a.catalog-tile-button(href=postURL) Open Thread if post.subject span: a.no-decoration.post-subject(href=postURL) #{post.subject} .catalog-tile-content if post.files.length > 0 .post-file-src - a(href=`/${board._id}/thread/${post.postId}#${post.postId}`) + a(href=postURL) if post.spoiler object(data='/img/spoiler.png' width='64' height='64') else diff --git a/views/mixins/post.pug b/views/mixins/post.pug index 1116335d..da98c7f9 100644 --- a/views/mixins/post.pug +++ b/views/mixins/post.pug @@ -1,6 +1,6 @@ mixin post(post, truncate, manage=false, globalmanage=false) article(id=post.postId class='post-container '+(post.thread ? '' : 'op')) - - const postURL = `/${post.board}/thread/${post.thread || post.postId}#${post.postId}`; + - const postURL = `/${post.board}/thread/${post.thread || post.postId}.html#${post.postId}`; header.post-info if globalmanage input.post-check(type='checkbox', name='globalcheckedposts[]' value=post._id) diff --git a/views/pages/board.pug b/views/pages/board.pug index 7bf5447f..8aa26cb6 100644 --- a/views/pages/board.pug +++ b/views/pages/board.pug @@ -12,7 +12,7 @@ block content include ../includes/pages.pug a(href='#bottom') [Bottom] | - a(href=`/${board._id}/catalog`) [Catalog] + a(href=`/${board._id}/catalog.html`) [Catalog] hr(size=1) form(action='/forms/board/'+board._id+'/actions' method='POST' enctype='application/x-www-form-urlencoded') input(type='hidden' name='_csrf' value=csrf) @@ -28,5 +28,5 @@ block content nav.pages#bottom a(href='#top') [Top] | - a(href=`/${board._id}/catalog`) [Catalog] + a(href=`/${board._id}/catalog.html`) [Catalog] include ../includes/actionfooter.pug diff --git a/views/pages/catalog.pug b/views/pages/catalog.pug index fbfa6a2b..e8ed7379 100644 --- a/views/pages/catalog.pug +++ b/views/pages/catalog.pug @@ -9,7 +9,7 @@ block content nav.pages#top a(href='#bottom') [Bottom] | - a(href=`/${board._id}/index`) [Return] + a(href=`/${board._id}/index.html`) [Return] hr(size=1) if threads.length === 0 p No posts. @@ -20,4 +20,4 @@ block content nav.pages#bottom a(href='#top') [Top] | - a(href=`/${board._id}/index`) [Return] + a(href=`/${board._id}/index.html`) [Return] diff --git a/views/pages/login.pug b/views/pages/login.pug index 9ea16120..b9f19ca2 100644 --- a/views/pages/login.pug +++ b/views/pages/login.pug @@ -6,8 +6,7 @@ block head block content section.form-wrapper.flex-center.mv-10 form.form-post(action='/forms/login' method='POST') - input(type='hidden' name='_csrf' value=csrf) - input(type='hidden' name='redirect' value=redirect) + //input(type='hidden' name='_csrf' value=csrf) section.postform-row .postform-label Username input#username(type='text', name='username', maxlength='50') @@ -15,5 +14,5 @@ block content .postform-label Password input#password(type='password', name='password', maxlength='100') input(type='submit', value='submit') - p No account? #[a(href='/register') Register] + p No account? #[a(href='/register.html') Register] diff --git a/views/pages/register.pug b/views/pages/register.pug index 0824a292..e57f0c99 100644 --- a/views/pages/register.pug +++ b/views/pages/register.pug @@ -22,4 +22,4 @@ block content img.captcha(src='/captcha' width=200 height=80) input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') input(type='submit', value='Register') - p Already have an account? #[a(href='/login') Login] + p Already have an account? #[a(href='/login.html') Login] diff --git a/views/pages/thread.pug b/views/pages/thread.pug index 71b8d363..8a773247 100644 --- a/views/pages/thread.pug +++ b/views/pages/thread.pug @@ -16,9 +16,9 @@ block content nav.pages#top a(href='#bottom') [Bottom] | - a(href=`/${board._id}/index`) [Return] + a(href=`/${board._id}/index.html`) [Return] | - a(href=`/${board._id}/catalog`) [Catalog] + a(href=`/${board._id}/catalog.html`) [Catalog] hr(size=1) form(action=`/forms/board/${board._id}/actions` method='POST' enctype='application/x-www-form-urlencoded') input(type='hidden' name='_csrf' value=csrf) @@ -30,7 +30,7 @@ block content nav.pages#bottom a(href='#top') [Top] | - a(href=`/${board._id}/index`) [Return] + a(href=`/${board._id}/index.html`) [Return] | - a(href=`/${board._id}/catalog`) [Catalog] + a(href=`/${board._id}/catalog.html`) [Catalog] include ../includes/actionfooter.pug diff --git a/wipe.js b/wipe.js index 0446e250..817f4e90 100644 --- a/wipe.js +++ b/wipe.js @@ -24,14 +24,16 @@ const Mongo = require(__dirname+'/db/db.js') console.log('deleting posts') await Posts.deleteAll('pol'); await Posts.deleteAll('b'); + await Posts.deleteAll('t'); console.log('deleting boards') await Boards.deleteIncrement('pol'); - await Boards.deleteIncrement('b'); + await Boards.deleteIncrement('b'); + await Boards.deleteIncrement('t'); await Boards.deleteAll(); await Trips.deleteAll(); console.log('deleting bans'); await Bans.deleteAll(); - console.log('adding b and pol') + console.log('adding boards') await Boards.insertOne({ _id: 'pol', name: 'Politically Incorrect', @@ -64,6 +66,22 @@ const Mongo = require(__dirname+'/db/db.js') defaultName: 'Anonymous', } }) + await Boards.insertOne({ + _id: 't', + name: 'text', + description: 'text only board', + owner: '', + moderators: [], + banners: [], + settings: { + forceAnon: true, + ids: false, + threadLimit: 100, + replyLimit: 300, + maxFiles: 0, + defaultName: 'Anonymous', + } + }) console.log('creating indexes') await Bans.db.dropIndexes(); await Bans.db.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 0 }); From 71974130e41352ba8ff3221ca499b3c648861c0d Mon Sep 17 00:00:00 2001 From: fatchan Date: Wed, 8 May 2019 19:41:54 +0000 Subject: [PATCH 02/27] catalog sorted now and tiles include sticky/lock/permasage icons --- db/posts.js | 3 +++ views/mixins/catalogtile.pug | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/db/posts.js b/db/posts.js index fa8aba1b..2f15cd9b 100644 --- a/db/posts.js +++ b/db/posts.js @@ -188,6 +188,9 @@ module.exports = { 'reports': 0, 'globalreports': 0, } + }).sort({ + 'sticky': -1, + 'bumped': -1, }).toArray(); }, diff --git a/views/mixins/catalogtile.pug b/views/mixins/catalogtile.pug index 2edd4235..24155d6a 100644 --- a/views/mixins/catalogtile.pug +++ b/views/mixins/catalogtile.pug @@ -13,11 +13,18 @@ mixin catalogtile(board, post, truncate) else object.catalog-thumb(data=`/img/thumb-${post.files[0].filename.split('.')[0]}.jpg` width='64' height='64') header.post-info - span: a(href=postURL) ##{post.postId} + if post.sticky + img(src='/img/sticky.svg' height='12') + if post.saged + img(src='/img/saged.svg' height='12') + if post.locked + img(src='/img/locked.svg' height='12') + | + span: a(href=postURL) No.#{post.postId} | span Replies: #{post.replyposts} | span Images: #{post.replyfiles} - br if post.message + br blockquote.no-m-p.post-message !{post.message} From fad5d045777509cbace36d338554d6402976c881 Mon Sep 17 00:00:00 2001 From: fatchan Date: Wed, 8 May 2019 19:44:47 +0000 Subject: [PATCH 03/27] fix rebuilding for first page bumps and implement basic rebuilding for post actions --- models/forms/actionhandler.js | 34 ++++++++++++++++++++++++++++++++-- models/forms/make-post.js | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/models/forms/actionhandler.js b/models/forms/actionhandler.js index 572d35a8..2fd500ac 100644 --- a/models/forms/actionhandler.js +++ b/models/forms/actionhandler.js @@ -14,7 +14,9 @@ const Posts = require(__dirname+'/../../db/posts.js') , dismissReports = require(__dirname+'/dismiss-report.js') , dismissGlobalReports = require(__dirname+'/dismissglobalreport.js') , actionChecker = require(__dirname+'/../../helpers/actionchecker.js') - , checkPerms = require(__dirname+'/../../helpers/hasperms.js'); + , checkPerms = require(__dirname+'/../../helpers/hasperms.js') + , remove = require('fs-extra').remove + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { @@ -226,8 +228,8 @@ module.exports = async (req, res, next) => { ) } await Promise.all(dbPromises); + const threadsToUpdate = [...new Set(posts.filter(post => post.thread !== null))]; if (aggregateNeeded) { - const threadsToUpdate = [...new Set(posts.filter(post => post.thread !== null))]; //recalculate and set correct aggregation numbers again await Promise.all(threadsToUpdate.map(async (post) => { const replyCounts = await Posts.getReplyCounts(post.board, post.thread); @@ -240,6 +242,34 @@ module.exports = async (req, res, next) => { Posts.setReplyCounts(post.board, post.thread, replyposts, replyfiles); })); } + + //now we need to delete outdated html + //TODO: not do this for reports, handle global actions & move to separate handler + optimize and test + const removePromises = [] + const postThreadIds = threadsToUpdate.map(x => x.thread); + const oldestThread = await Posts.db.find({ + 'thread': null, + 'board': req.params.board, + 'postId': { + '$in': postThreadIds + } + }).sort({ + 'bumped': 1 + }).limit(1).toArray(); + //always need to refresh catalog + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/catalog.html`)); + //refresh any impacted threads + for (let i = 0; i < threadsToUpdate.length; i++) { + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${threadsToUpdate[i].thread}.html`)); + } + //refersh all pages above oldest thread affected + const numThreadsBefore = await Posts.getBeforeCount(req.params.board, oldestThread); + const pagesToRemove = Math.ceil(numThreadsBefore/10) || 1; + for (let i = 1; i <= pagesToRemove; i++) { + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); + } + await Promise.all(removePromises); + } catch (err) { return next(err); } diff --git a/models/forms/make-post.js b/models/forms/make-post.js index 611c9926..02bf0a66 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -237,7 +237,7 @@ module.exports = async (req, res, next, numFiles) => { if (!data.sage) { //bumping a thread, so delete all pages above it const numThreadsBefore = await Posts.getBeforeCount(req.params.board, thread); - const pagesToRemove = Math.ceil(numThreadsBefore/10); + const pagesToRemove = Math.ceil(numThreadsBefore/10) || 1; //|| 1, so we always refresh first page incase this is the top thread nothing will be before it for (let i = 1; i <= pagesToRemove; i++) { removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); } From f800c7e20580725e3de0d699f972db9a93d82ef1 Mon Sep 17 00:00:00 2001 From: fatchan Date: Thu, 9 May 2019 05:12:48 +0000 Subject: [PATCH 04/27] refresh page that thread preview shown on when new sage post is made --- models/forms/make-post.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/models/forms/make-post.js b/models/forms/make-post.js index 02bf0a66..29f110e6 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -234,11 +234,14 @@ module.exports = async (req, res, next, numFiles) => { if (data.thread) { //refresh the thread itself removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${req.body.thread}.html`)); + //refersh pages + const numThreadsBefore = await Posts.getBeforeCount(req.params.board, thread); + const pagesToRemove = Math.ceil(numThreadsBefore/10) || 1; + //refresh the page that the thread is on + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${pagesToRemove == 1 ? 'index' : pagesToRemove}.html`)); if (!data.sage) { - //bumping a thread, so delete all pages above it - const numThreadsBefore = await Posts.getBeforeCount(req.params.board, thread); - const pagesToRemove = Math.ceil(numThreadsBefore/10) || 1; //|| 1, so we always refresh first page incase this is the top thread nothing will be before it - for (let i = 1; i <= pagesToRemove; i++) { + //if not saged, it will bump so we should refresh any pages above it as well + for (let i = pagesToRemove-1; i >= 1; i--) { removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); } } From 77a0b6ecb92b0e16ecd0139f0bf8fb369a9b1458 Mon Sep 17 00:00:00 2001 From: fatchan Date: Fri, 10 May 2019 10:43:46 +0000 Subject: [PATCH 05/27] changes to handing actions better --- controllers/pages.js | 2 +- db/posts.js | 17 +++- models/forms/actionhandler.js | 175 ++++++++++++++++++++++++++-------- models/forms/make-post.js | 11 ++- models/forms/stickyposts.js | 3 +- models/pages/board.js | 5 +- views/mixins/post.pug | 2 +- 7 files changed, 163 insertions(+), 52 deletions(-) diff --git a/controllers/pages.js b/controllers/pages.js index 5968e4c3..5e3e97c8 100644 --- a/controllers/pages.js +++ b/controllers/pages.js @@ -54,7 +54,7 @@ router.get('/:board/manage.html', Boards.exists, isLoggedIn, hasPerms, csrf, man router.get('/globalmanage.html', isLoggedIn, hasPerms, csrf, globalManage); // board page/recents -router.get('/:board/:page([2-9]*|index).html', Boards.exists, paramConverter, board); +router.get('/:board/:page(1[0-9]*|[2-9]*|index).html', Boards.exists, paramConverter, board); // thread view page router.get('/:board/thread/:id(\\d+).html', Boards.exists, paramConverter, thread); diff --git a/db/posts.js b/db/posts.js index 2f15cd9b..f856803b 100644 --- a/db/posts.js +++ b/db/posts.js @@ -14,7 +14,17 @@ module.exports = { 'board': board, 'thread': null, 'bumped': { - '$gt': thread.bumped + '$gte': thread.bumped + } + }); + }, + + getAfterCount: (board, thread) => { + return db.countDocuments({ + 'board': board, + 'thread': null, + 'bumped': { + '$lte': thread.bumped } }); }, @@ -347,7 +357,7 @@ module.exports = { }).skip(threadLimit).toArray(); //if there are any if (threads.length === 0) { - return; + return []; } //get the postIds const threadIds = threads.map(thread => thread.postId); @@ -365,7 +375,8 @@ module.exports = { } //get the mongoIds and delete them all const postMongoIds = postsAndThreads.map(post => Mongo.ObjectId(post._id)); - return module.exports.deleteMany(postMongoIds); + await module.exports.deleteMany(postMongoIds); + return threadIds; }, deleteMany: (ids) => { diff --git a/models/forms/actionhandler.js b/models/forms/actionhandler.js index 2fd500ac..bd73efd6 100644 --- a/models/forms/actionhandler.js +++ b/models/forms/actionhandler.js @@ -63,7 +63,7 @@ module.exports = async (req, res, next) => { }) } - const posts = await Posts.getPosts(req.params.board, req.body.checkedposts, true); + let posts = await Posts.getPosts(req.params.board, req.body.checkedposts, true); if (!posts || posts.length === 0) { return res.status(404).render('message', { 'title': 'Not found', @@ -130,6 +130,7 @@ module.exports = async (req, res, next) => { query['board'] = req.params.board; } const deleteIpPosts = await Posts.db.find(query).toArray(); + posts = posts.concat(deleteIpPosts); if (deleteIpPosts && deleteIpPosts.length > 0) { const { message } = await deletePosts(req, res, next, deleteIpPosts, req.params.board); messages.push(message); @@ -208,30 +209,40 @@ module.exports = async (req, res, next) => { messages.push(message); } } - const dbPromises = [] + const bulkWrites = [] if (Object.keys(combinedQuery).length > 0) { - dbPromises.push( - Posts.db.updateMany({ - '_id': { - '$in': postMongoIds - } - }, combinedQuery) - ) + bulkWrites.push({ + 'updateMany': { + 'filter': { + '_id': { + '$in': postMongoIds + } + }, + 'update': combinedQuery + } + }); } if (Object.keys(passwordCombinedQuery).length > 0) { - dbPromises.push( - Posts.db.updateMany({ - '_id': { - '$in': passwordPostMongoIds - } - }, passwordCombinedQuery) - ) + bulkWrites.push({ + 'updateMany': { + 'filter': { + '_id': { + '$in': passwordPostMongoIds + } + }, + 'update': passwordCombinedQuery + } + }); } - await Promise.all(dbPromises); - const threadsToUpdate = [...new Set(posts.filter(post => post.thread !== null))]; + if (bulkWrites.length > 0) { + await Posts.db.bulkWrite(bulkWrites); + } + + //get only posts (so we can use them for thread ids + const postThreadsToUpdate = posts.filter(post => post.thread !== null); if (aggregateNeeded) { - //recalculate and set correct aggregation numbers again - await Promise.all(threadsToUpdate.map(async (post) => { + //recalculate replies and image counts + await Promise.all(postThreadsToUpdate.map(async (post) => { const replyCounts = await Posts.getReplyCounts(post.board, post.thread); let replyposts = 0; let replyfiles = 0; @@ -243,30 +254,114 @@ module.exports = async (req, res, next) => { })); } + //map thread ids to board + const boardThreadMap = {}; + const queryOrs = []; + for (let i = 0; i < posts.length; i++) { + const post = posts[i]; + if (!boardThreadMap[post.board]) { + boardThreadMap[post.board] = []; + } + boardThreadMap[post.board].push(post.thread || post.postId); + } + + //make it into an OR query for the db + const threadBoards = Object.keys(boardThreadMap); + for (let i = 0; i < threadBoards.length; i++) { + const threadBoard = threadBoards[i]; + boardThreadMap[threadBoard] = [...new Set(boardThreadMap[threadBoard])] + queryOrs.push({ + 'board': threadBoard, + 'postId': { + '$in': boardThreadMap[threadBoard] + } + }) + } + + //fetch threads per board that we only checked posts for + let threadsEachBoard = await Posts.db.find({ + 'thread': null, + '$or': queryOrs + }).toArray(); + //combine it with what we already had + threadsEachBoard = threadsEachBoard.concat(posts.filter(post => post.thread === null)) + + //get the oldest and newest thread for each board to determine how to delete + const threadBounds = threadsEachBoard.reduce((acc, curr) => { + if (!acc[curr.board] || curr.bumped < acc[curr.board].bumped) { + acc[curr.board] = { oldest: null, newest: null}; + } + if (!acc[curr.board].oldest || curr.bumped < acc[curr.board].oldest.bumped) { + acc[curr.board].oldest = curr; + } + if (!acc[curr.board].newest || curr.bumped > acc[curr.board].newest.bumped) { + acc[curr.board].newest = curr; + } + return acc; + }, {}); + +//console.log(threadBounds); +//console.log(boardThreadMap); //now we need to delete outdated html //TODO: not do this for reports, handle global actions & move to separate handler + optimize and test const removePromises = [] - const postThreadIds = threadsToUpdate.map(x => x.thread); - const oldestThread = await Posts.db.find({ - 'thread': null, - 'board': req.params.board, - 'postId': { - '$in': postThreadIds + const boardsWithChanges = Object.keys(threadBounds); + for (let i = 0; i < boardsWithChanges.length; i++) { + const changeBoard = boardsWithChanges[i]; + const bounds = threadBounds[changeBoard]; +//console.log(changeBoard, 'OLDEST thread', bounds.oldest.postId) +//console.log(changeBoard, 'NEWEST thread', bounds.newest.postId) + //always need to refresh catalog + removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/catalog.html`)); + //refresh any impacted threads + for (let j = 0; j < boardThreadMap[changeBoard].length; j++) { + removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/thread/${boardThreadMap[changeBoard][j]}.html`)); +//console.log(changeBoard, 'update thread', boardThreadMap[changeBoard][j]); + } + //refersh all pages affected + const maxPages = Math.ceil((await Posts.getPages(changeBoard)) / 10); + let pagesToRemoveAfter; + let pagesToRemoveBefore; + if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) { //deletes only affect later pages as they end up higher up + //remove current and later pages for deletes + pagesToRemoveAfter = Math.ceil((await Posts.getAfterCount(changeBoard, bounds.newest))/10) || 1; +//console.log(changeBoard, 'pages to remove after newest affected thread', pagesToRemoveAfter); + for (let j = maxPages; j > maxPages-pagesToRemoveAfter; j--) { +//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) + removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); + } + } else if (req.body.sticky) { //use an else if because if deleting, other actions are not executed/irrelevant + //remove current and newer pages for stickies + pagesToRemoveBefore = Math.ceil((await Posts.getBeforeCount(changeBoard, bounds.oldest))/10) || 1; +//console.log(changeBoard, 'pages to remove before oldest affected thread', pagesToRemoveBefore); + for (let j = 1; j <= pagesToRemoveBefore; j++) { +//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) + removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); + } + } else if ((hasPerms && (req.body.lock || req.body.sage)) || req.body.spoiler) { //these actions do not affect other pages + //remove only pages with affected threads + if (!pagesToRemoveBefore) { + pagesToRemoveBefore = Math.ceil((await Posts.getBeforeCount(changeBoard, bounds.oldest))/10) || 1; + } + if (!pagesToRemoveAfter) { + pagesToRemoveAfter = Math.ceil((await Posts.getAfterCount(changeBoard, bounds.newest))/10) || 1; + } +//console.log(changeBoard, 'remove all inbetween pages because finding affected pages is hard at 1am', pagesToRemoveBefore, pagesToRemoveAfter) + //console.log(pagesToRemoveBefore, pagesToRemoveAfter, maxPages) + for (let j = pagesToRemoveBefore; j <= maxPages-pagesToRemoveAfter+1; j++) { + //console.log(j) +//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) + removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); + } + } + const maxPagesAfter = Math.ceil((await Posts.getPages(changeBoard)) / 10); + if (maxPages !== maxPagesAfter) { + // number of pages changed, delete all pages (because of page number buttons on existing pages) + for (let j = 1; j <= maxPages; j++) { +//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) + removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); + } } - }).sort({ - 'bumped': 1 - }).limit(1).toArray(); - //always need to refresh catalog - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/catalog.html`)); - //refresh any impacted threads - for (let i = 0; i < threadsToUpdate.length; i++) { - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${threadsToUpdate[i].thread}.html`)); - } - //refersh all pages above oldest thread affected - const numThreadsBefore = await Posts.getBeforeCount(req.params.board, oldestThread); - const pagesToRemove = Math.ceil(numThreadsBefore/10) || 1; - for (let i = 1; i <= pagesToRemove; i++) { - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); } await Promise.all(removePromises); diff --git a/models/forms/make-post.js b/models/forms/make-post.js index 29f110e6..4545a489 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -222,13 +222,16 @@ module.exports = async (req, res, next, numFiles) => { } const postId = await Posts.insertOne(req.params.board, data, thread); - if (!data.thread) { - //if we just added a new thread, prune any old ones - await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); - } //now we need to delete outdated html const removePromises = [] + if (!data.thread) { + //if we just added a new thread, prune any old ones + const prunedThreads = await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); + for (let i = 0; i < prunedThreads.length; i++) { + removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`)); + } + } //always need to refresh catalog removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/catalog.html`)); if (data.thread) { diff --git a/models/forms/stickyposts.js b/models/forms/stickyposts.js index 2b50a80a..5b32f0ee 100644 --- a/models/forms/stickyposts.js +++ b/models/forms/stickyposts.js @@ -16,7 +16,8 @@ module.exports = (posts) => { message: `Stickied ${filteredposts.length} post(s)`, action: '$set', query: { - 'sticky': true + 'sticky': true, + 'bumped': 8640000000000000 } }; diff --git a/models/pages/board.js b/models/pages/board.js index d0c021f0..0db052e0 100644 --- a/models/pages/board.js +++ b/models/pages/board.js @@ -6,7 +6,8 @@ const Posts = require(__dirname+'/../../db/posts.js') module.exports = async (req, res, next) => { - const page = req.params.page === 'index' ? 1 : (req.params.page || 1); + const page = req.params.page === 'index' ? 1 : req.params.page; + const pageName = page === 1 ? 'index' : page; let threads; let pages; let pageURL; @@ -16,7 +17,7 @@ module.exports = async (req, res, next) => { return next(); } threads = await Posts.getRecent(req.params.board, page); - pageURL = `${req.params.board}/${req.params.page}.html`; + pageURL = `${req.params.board}/${pageName}.html`; await writePageHTML(pageURL, 'board.pug', { board: res.locals.board, threads: threads || [], diff --git a/views/mixins/post.pug b/views/mixins/post.pug index da98c7f9..0d9785ee 100644 --- a/views/mixins/post.pug +++ b/views/mixins/post.pug @@ -32,7 +32,7 @@ mixin post(post, truncate, manage=false, globalmanage=false) | span #{post.date.toLocaleString()} | - if post.userId && board.settings.ids + if post.userId span.user-id(style=`background: #${post.userId}`) #{post.userId} | span: a(href=postURL) No.#{post.postId} From 5158893bbc4b6121aba0c36ac1944492bf92ab14 Mon Sep 17 00:00:00 2001 From: fatchan Date: Fri, 10 May 2019 19:51:13 +0000 Subject: [PATCH 06/27] move to an immediate build strategy to prevent issues with active pages rebuilding excessively on quick successive request --- build.js | 91 +++++++++++++++++++ controllers/forms.js | 2 + controllers/pages.js | 3 +- db/posts.js | 29 +++--- helpers/{writepagehtml.js => render.js} | 6 +- models/forms/actionhandler.js | 115 +++++++++++------------- models/forms/delete-post.js | 8 ++ models/forms/make-post.js | 46 +++++----- models/pages/board.js | 23 ++--- models/pages/catalog.js | 17 +--- models/pages/changepassword.js | 4 +- models/pages/home.js | 10 +-- models/pages/login.js | 4 +- models/pages/manage.js | 1 - models/pages/register.js | 4 +- models/pages/thread.js | 26 ++---- views/includes/pages.pug | 2 +- 17 files changed, 218 insertions(+), 173 deletions(-) create mode 100644 build.js rename helpers/{writepagehtml.js => render.js} (54%) diff --git a/build.js b/build.js new file mode 100644 index 00000000..12e6ab27 --- /dev/null +++ b/build.js @@ -0,0 +1,91 @@ +'use strict'; + +const Posts = require(__dirname+'/db/posts.js') + , Boards = require(__dirname+'/db/boards.js') + , uploadDirectory = require(__dirname+'/helpers/uploadDirectory.js') + , render = require(__dirname+'/helpers/render.js'); + +module.exports = { + + buildCatalog: async (board) => { + const threads = await Posts.getCatalog(board._id); + return render(`${board._id}/catalog.html`, 'catalog.pug', { + board, + threads + }); + }, + + buildThread: async (threadId, board) => { +//console.log('building thread', `${board._id}/thread/${threadId}.html`); + if (!board._id) { + board = await Boards.findOne(board); + } + const thread = await Posts.getThread(board._id, threadId) + if (!thread) { + return; //this thread may have been an OP that was deleted during a rebuild + } + return render(`${board._id}/thread/${threadId}.html`, 'thread.pug', { + board, + thread + }); + }, + + buildBoard: async (board, page, maxPage=null) => { +//console.log('building board page', `${board._id}/${page === 1 ? 'index' : page}.html`); + const threads = await Posts.getRecent(board._id, page); + if (!maxPage) { + maxPage = Math.ceil((await Posts.getPages(board._id)) / 10); + } + return render(`${board._id}/${page === 1 ? 'index' : page}.html`, 'board.pug', { + board, + threads, + maxPage, + page + }); + }, + + //building multiple pages (for rebuilds) + buildBoardMultiple: async (board, startpage=1, endpage=10) => { + const maxPage = Math.ceil((await Posts.getPages(board._id)) / 10); + endpage = maxPage < endpage ? maxPage : endpage; + const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0 + const threads = await Posts.getRecent(board._id, startpage, difference*10); + const buildArray = []; + for (let i = startpage; i <= endpage; i++) { +//console.log('multi building board page', `${board._id}/${i === 1 ? 'index' : i}.html`); + let spliceStart = (i-1)*10; + if (spliceStart > 0) { + spliceStart = spliceStart - 1; + } + buildArray.push( + render(`${board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', { + board, + threads: threads.splice(0,10), + maxPage, + page: i, + }) + ); + } + return Promise.all(buildArray); + }, + + buildHomepage: async () => { + const boards = await Boards.find(); + return render('index.html', 'home.pug', { + boards + }); + }, + + buildChangePassword: () => { + return render('changepassword.html', 'changepassword.pug'); + }, + + buildLogin: () => { + return render('login.html', 'login.pug'); + }, + + buildRegister: () => { + return render('register.html', 'register.pug'); + }, + +} diff --git a/controllers/forms.js b/controllers/forms.js index a7e4bd4f..48f56642 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -8,6 +8,8 @@ const express = require('express') , Trips = require(__dirname+'/../db/trips.js') , Bans = require(__dirname+'/../db/bans.js') , Mongo = require(__dirname+'/../db/db.js') + , deletePosts = require(__dirname+'/../models/forms/delete-post.js') + , dismissGlobaReports = require(__dirname+'/../models/forms/dismissglobalreport.js') , banPoster = require(__dirname+'/../models/forms/ban-poster.js') , removeBans = require(__dirname+'/../models/forms/removebans.js') , makePost = require(__dirname+'/../models/forms/make-post.js') diff --git a/controllers/pages.js b/controllers/pages.js index 5e3e97c8..4b3e0c66 100644 --- a/controllers/pages.js +++ b/controllers/pages.js @@ -3,6 +3,7 @@ const express = require('express') , router = express.Router() , Boards = require(__dirname+'/../db/boards.js') + , Posts = require(__dirname+'/../db/posts.js') , hasPerms = require(__dirname+'/../helpers/haspermsmiddleware.js') , isLoggedIn = require(__dirname+'/../helpers/isloggedin.js') , paramConverter = require(__dirname+'/../helpers/paramconverter.js') @@ -57,7 +58,7 @@ router.get('/globalmanage.html', isLoggedIn, hasPerms, csrf, globalManage); router.get('/:board/:page(1[0-9]*|[2-9]*|index).html', Boards.exists, paramConverter, board); // thread view page -router.get('/:board/thread/:id(\\d+).html', Boards.exists, paramConverter, thread); +router.get('/:board/thread/:id(\\d+).html', Boards.exists, paramConverter, Posts.exists, thread); // board catalog page router.get('/:board/catalog.html', Boards.exists, catalog); diff --git a/db/posts.js b/db/posts.js index f856803b..3744cd24 100644 --- a/db/posts.js +++ b/db/posts.js @@ -9,27 +9,18 @@ module.exports = { db, - getBeforeCount: (board, thread) => { - return db.countDocuments({ + getThreadPage: async (board, thread) => { + const threadsBefore = await db.countDocuments({ 'board': board, 'thread': null, 'bumped': { '$gte': thread.bumped } }); + return Math.ceil(threadsBefore/10) || 1; //1 because 0 threads before is page 1 }, - getAfterCount: (board, thread) => { - return db.countDocuments({ - 'board': board, - 'thread': null, - 'bumped': { - '$lte': thread.bumped - } - }); - }, - - getRecent: async (board, page) => { + getRecent: async (board, page, limit=10) => { // get all thread posts (posts with null thread id) const threads = await db.find({ 'thread': null, @@ -45,7 +36,7 @@ module.exports = { }).sort({ 'sticky': -1, 'bumped': -1, - }).skip(10*(page-1)).limit(10).toArray(); + }).skip(10*(page-1)).limit(limit).toArray(); // add last 5 posts in reverse order to preview await Promise.all(threads.map(async thread => { @@ -145,7 +136,6 @@ module.exports = { thread.replies = data[1]; } return thread; - }, getThreadPosts: (board, id) => { @@ -393,4 +383,13 @@ module.exports = { }); }, + exists: async (req, res, next) => { + const thread = await module.exports.getThread(req.params.board, req.params.id); + if (!thread) { + return res.status(404).render('404'); + } + res.locals.thread = thread; // can acces this in views or next route handlers + next(); + } + } diff --git a/helpers/writepagehtml.js b/helpers/render.js similarity index 54% rename from helpers/writepagehtml.js rename to helpers/render.js index 3880c931..3708a102 100644 --- a/helpers/writepagehtml.js +++ b/helpers/render.js @@ -4,9 +4,9 @@ const outputFile = require('fs-extra').outputFile , pug = require('pug') , path = require('path') , uploadDirectory = require(__dirname+'/uploadDirectory.js') - , pugDirectory = path.join(__dirname+'/../views/pages/'); + , templateDirectory = path.join(__dirname+'/../views/pages/'); -module.exports = async (htmlName, pugName, pugVars) => { - const html = pug.renderFile(`${pugDirectory}${pugName}`, pugVars); +module.exports = async (htmlName, templateName, options) => { + const html = pug.renderFile(`${templateDirectory}${templateName}`, options); return outputFile(`${uploadDirectory}html/${htmlName}`, html); }; diff --git a/models/forms/actionhandler.js b/models/forms/actionhandler.js index bd73efd6..d876ed0d 100644 --- a/models/forms/actionhandler.js +++ b/models/forms/actionhandler.js @@ -16,7 +16,8 @@ const Posts = require(__dirname+'/../../db/posts.js') , actionChecker = require(__dirname+'/../../helpers/actionchecker.js') , checkPerms = require(__dirname+'/../../helpers/hasperms.js') , remove = require('fs-extra').remove - , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') + , { buildCatalog, buildThread, buildBoardMultiple } = require(__dirname+'/../../build.js'); module.exports = async (req, res, next) => { @@ -234,6 +235,29 @@ module.exports = async (req, res, next) => { } }); } + + //get a map of boards to threads affected + const boardThreadMap = {}; + const queryOrs = []; + for (let i = 0; i < posts.length; i++) { + const post = posts[i]; + if (!boardThreadMap[post.board]) { + boardThreadMap[post.board] = []; + } + boardThreadMap[post.board].push(post.thread || post.postId); + } + + const beforePages = {}; + const threadBoards = Object.keys(boardThreadMap); + //get how many pages each board is to know whether we should rebuild all pages (because of page nav changes) + //only if deletes actions selected because this could result in number of pages to change + if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) { + await Promise.all(threadBoards.map(async board => { + beforePages[board] = Math.ceil((await Posts.getPages(board)) / 10); + })); + } + + //execute actions now if (bulkWrites.length > 0) { await Posts.db.bulkWrite(bulkWrites); } @@ -254,19 +278,7 @@ module.exports = async (req, res, next) => { })); } - //map thread ids to board - const boardThreadMap = {}; - const queryOrs = []; - for (let i = 0; i < posts.length; i++) { - const post = posts[i]; - if (!boardThreadMap[post.board]) { - boardThreadMap[post.board] = []; - } - boardThreadMap[post.board].push(post.thread || post.postId); - } - //make it into an OR query for the db - const threadBoards = Object.keys(boardThreadMap); for (let i = 0; i < threadBoards.length; i++) { const threadBoard = threadBoards[i]; boardThreadMap[threadBoard] = [...new Set(boardThreadMap[threadBoard])] @@ -300,70 +312,43 @@ module.exports = async (req, res, next) => { return acc; }, {}); -//console.log(threadBounds); -//console.log(boardThreadMap); //now we need to delete outdated html //TODO: not do this for reports, handle global actions & move to separate handler + optimize and test - const removePromises = [] + const parallelPromises = [] const boardsWithChanges = Object.keys(threadBounds); for (let i = 0; i < boardsWithChanges.length; i++) { const changeBoard = boardsWithChanges[i]; const bounds = threadBounds[changeBoard]; -//console.log(changeBoard, 'OLDEST thread', bounds.oldest.postId) -//console.log(changeBoard, 'NEWEST thread', bounds.newest.postId) //always need to refresh catalog - removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/catalog.html`)); - //refresh any impacted threads + parallelPromises.push(buildCatalog(res.locals.board)); + //rebuild impacted threads for (let j = 0; j < boardThreadMap[changeBoard].length; j++) { - removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/thread/${boardThreadMap[changeBoard][j]}.html`)); -//console.log(changeBoard, 'update thread', boardThreadMap[changeBoard][j]); - } - //refersh all pages affected - const maxPages = Math.ceil((await Posts.getPages(changeBoard)) / 10); - let pagesToRemoveAfter; - let pagesToRemoveBefore; - if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) { //deletes only affect later pages as they end up higher up - //remove current and later pages for deletes - pagesToRemoveAfter = Math.ceil((await Posts.getAfterCount(changeBoard, bounds.newest))/10) || 1; -//console.log(changeBoard, 'pages to remove after newest affected thread', pagesToRemoveAfter); - for (let j = maxPages; j > maxPages-pagesToRemoveAfter; j--) { -//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) - removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); - } - } else if (req.body.sticky) { //use an else if because if deleting, other actions are not executed/irrelevant - //remove current and newer pages for stickies - pagesToRemoveBefore = Math.ceil((await Posts.getBeforeCount(changeBoard, bounds.oldest))/10) || 1; -//console.log(changeBoard, 'pages to remove before oldest affected thread', pagesToRemoveBefore); - for (let j = 1; j <= pagesToRemoveBefore; j++) { -//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) - removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); - } - } else if ((hasPerms && (req.body.lock || req.body.sage)) || req.body.spoiler) { //these actions do not affect other pages - //remove only pages with affected threads - if (!pagesToRemoveBefore) { - pagesToRemoveBefore = Math.ceil((await Posts.getBeforeCount(changeBoard, bounds.oldest))/10) || 1; - } - if (!pagesToRemoveAfter) { - pagesToRemoveAfter = Math.ceil((await Posts.getAfterCount(changeBoard, bounds.newest))/10) || 1; - } -//console.log(changeBoard, 'remove all inbetween pages because finding affected pages is hard at 1am', pagesToRemoveBefore, pagesToRemoveAfter) - //console.log(pagesToRemoveBefore, pagesToRemoveAfter, maxPages) - for (let j = pagesToRemoveBefore; j <= maxPages-pagesToRemoveAfter+1; j++) { - //console.log(j) -//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) - removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); - } + parallelPromises.push(buildThread(boardThreadMap[changeBoard][j], changeBoard)); } - const maxPagesAfter = Math.ceil((await Posts.getPages(changeBoard)) / 10); - if (maxPages !== maxPagesAfter) { - // number of pages changed, delete all pages (because of page number buttons on existing pages) - for (let j = 1; j <= maxPages; j++) { -//console.log(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`) - removePromises.push(remove(`${uploadDirectory}html/${changeBoard}/${j == 1 ? 'index' : j}.html`)); + //refersh any pages affected + const afterPages = Math.ceil((await Posts.getPages(changeBoard)) / 10); + if (beforePages[changeBoard] && beforePages[changeBoard] !== afterPages) { + //amount of pages changed, rebuild all pages + parallelPromises.push(buildBoardMultiple(res.locals.board, 1, afterPages)); + } else { + const threadPageOldest = await Posts.getThreadPage(req.params.board, bounds.oldest); + const threadPageNewest = await Posts.getThreadPage(req.params.board, bounds.newest); + if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) { + //rebuild current and older pages for deletes + parallelPromises.push(buildBoardMultiple(res.locals.board, threadPageNewest, afterPages)); + } else if (req.body.sticky) { //else if -- if deleting, other actions are not executed/irrelevant + //rebuild current and newer pages for stickies + parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPageOldest)); + } else if ((hasPerms && (req.body.lock || req.body.sage)) || req.body.spoiler) { + //rebuild inbewteen pages for things that dont cause page/thread movement + //should rebuild only affected pages, but finding the page of all affected + //threads could end up being slower/more resource intensive. this is simpler + //but still avoids rebuilding _some_ pages unnecessarily + parallelPromises.push(buildBoardMultiple(res.locals.board, threadPageNewest, threadPageOldest)); } } } - await Promise.all(removePromises); + await Promise.all(parallelPromises); } catch (err) { return next(err); diff --git a/models/forms/delete-post.js b/models/forms/delete-post.js index 586d115c..eee05592 100644 --- a/models/forms/delete-post.js +++ b/models/forms/delete-post.js @@ -2,6 +2,7 @@ const uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') , deletePostFiles = require(__dirname+'/../../helpers/files/deletepostfiles.js') + , remove = require('fs-extra').remove , Mongo = require(__dirname+'/../../db/db.js') , Posts = require(__dirname+'/../../db/posts.js'); @@ -10,6 +11,13 @@ module.exports = async (req, res, next, posts, board) => { //filter to threads const threads = posts.filter(x => x.thread == null); + //delete the html for threads + const deleteHTML = [] + for (let i = 0; i < threads.length; i++) { + deleteHTML.push(remove(`${uploadDirectory}html/${threads[i].board}/thread/${threads[i].postId}.html`)); + } + await Promise.all(deleteHTML); + //get posts from all threads let threadPosts = [] if (threads.length > 0) { diff --git a/models/forms/make-post.js b/models/forms/make-post.js index 4545a489..00991c73 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -27,7 +27,8 @@ const uuidv4 = require('uuid/v4') , imageIdentify = require(__dirname+'/../../helpers/files/image-identify.js') , videoThumbnail = require(__dirname+'/../../helpers/files/video-thumbnail.js') , videoIdentify = require(__dirname+'/../../helpers/files/video-identify.js') - , formatSize = require(__dirname+'/../../helpers/files/format-size.js'); + , formatSize = require(__dirname+'/../../helpers/files/format-size.js') + , { buildCatalog, buildThread, buildBoard, buildBoardMultiple } = require(__dirname+'/../../build.js'); module.exports = async (req, res, next, numFiles) => { @@ -153,9 +154,10 @@ module.exports = async (req, res, next, numFiles) => { } //forceanon hide reply subjects so cant be used as name for replies - let subject = hasPerms || !forceAnon || !req.body.thread ? req.body.subject : null; //forceanon only allow sage email - let email = hasPerms || !forceAnon || req.body.email === 'sage' ? req.body.email : null; + let subject = (hasPerms || !forceAnon || !req.body.thread) ? req.body.subject : null; + let email = (hasPerms || !forceAnon || req.body.email === 'sage') ? req.body.email : null; + let name = res.locals.board.settings.defaultName; let tripcode = null; let capcode = null; @@ -223,38 +225,32 @@ module.exports = async (req, res, next, numFiles) => { const postId = await Posts.insertOne(req.params.board, data, thread); - //now we need to delete outdated html - const removePromises = [] - if (!data.thread) { + //now we need to rebuild pages + const parallelPromises = [] + //always need to rebuild catalog + parallelPromises.push(buildCatalog(res.locals.board)); + if (data.thread) { //if we just added a new thread, prune any old ones const prunedThreads = await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); for (let i = 0; i < prunedThreads.length; i++) { - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`)); + parallelPromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`)); } - } - //always need to refresh catalog - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/catalog.html`)); - if (data.thread) { //refresh the thread itself - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${req.body.thread}.html`)); + parallelPromises.push(buildThread(thread.postId, res.locals.board)); //refersh pages - const numThreadsBefore = await Posts.getBeforeCount(req.params.board, thread); - const pagesToRemove = Math.ceil(numThreadsBefore/10) || 1; - //refresh the page that the thread is on - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${pagesToRemove == 1 ? 'index' : pagesToRemove}.html`)); - if (!data.sage) { + const threadPage = await Posts.getThreadPage(req.params.board, thread); + if (data.email === 'sage') { + //refresh the page that the thread is on + parallelPromises.push(buildBoard(res.locals.board, threadPage)); + } else { //if not saged, it will bump so we should refresh any pages above it as well - for (let i = pagesToRemove-1; i >= 1; i--) { - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); - } + parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPage)); } } else { - //new thread, remove all pages - for (let i = 1; i <= Math.ceil(res.locals.board.settings.threadLimit/10); i++) { - removePromises.push(remove(`${uploadDirectory}html/${req.params.board}/${i == 1 ? 'index' : i}.html`)); - } + //new thread, rebuild all pages + parallelPromises.push(buildBoardMultiple(res.locals.board, 1, 10)); } - await Promise.all(removePromises); + await Promise.all(parallelPromises); const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`; diff --git a/models/pages/board.js b/models/pages/board.js index 0db052e0..e2fa88d9 100644 --- a/models/pages/board.js +++ b/models/pages/board.js @@ -1,33 +1,22 @@ 'use strict'; const Posts = require(__dirname+'/../../db/posts.js') - , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') - , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); + , { buildBoard } = require(__dirname+'/../../build.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { const page = req.params.page === 'index' ? 1 : req.params.page; - const pageName = page === 1 ? 'index' : page; - let threads; - let pages; - let pageURL; try { - pages = Math.ceil((await Posts.getPages(req.params.board)) / 10) - if (page > pages && pages > 0) { + const maxPage = Math.ceil((await Posts.getPages(req.params.board)) / 10); + if (page > maxPage && maxPage > 0) { return next(); } - threads = await Posts.getRecent(req.params.board, page); - pageURL = `${req.params.board}/${pageName}.html`; - await writePageHTML(pageURL, 'board.pug', { - board: res.locals.board, - threads: threads || [], - pages, - page - }); + await buildBoard(res.locals.board, page, maxPage); } catch (err) { return next(err); } - return res.sendFile(`${uploadDirectory}html/${pageURL}`); + return res.sendFile(`${uploadDirectory}html/${req.params.board}/${page === 1 ? 'index' : page}.html`); } diff --git a/models/pages/catalog.js b/models/pages/catalog.js index 189550af..d818bb3d 100644 --- a/models/pages/catalog.js +++ b/models/pages/catalog.js @@ -1,25 +1,16 @@ 'use strict'; -const Posts = require(__dirname+'/../../db/posts.js') - , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') - , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); +const { buildCatalog } = require(__dirname+'/../../build.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { - // get all threads - let threads; - let pageURL; try { - threads = await Posts.getCatalog(req.params.board); - pageURL = `${req.params.board}/catalog.html`; - await writePageHTML(pageURL, 'catalog.pug', { - board: res.locals.board, - threads: threads || [], - }); + await buildCatalog(res.locals.board); } catch (err) { return next(err); } - return res.sendFile(`${uploadDirectory}html/${pageURL}`); + return res.sendFile(`${uploadDirectory}html/${req.params.board}/catalog.html`); } diff --git a/models/pages/changepassword.js b/models/pages/changepassword.js index 277d5329..ac3be547 100644 --- a/models/pages/changepassword.js +++ b/models/pages/changepassword.js @@ -1,12 +1,12 @@ 'use strict'; -const writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js') +const { buildChangePassword } = require(__dirname+'/../../build.js') , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { try { - await writePageHTML('changepassword.html', 'changepassword.pug'); + await buildChangePassword(); } catch (err) { return next(err); } diff --git a/models/pages/home.js b/models/pages/home.js index 7c722622..044cbbb0 100644 --- a/models/pages/home.js +++ b/models/pages/home.js @@ -1,16 +1,12 @@ 'use strict'; -const Boards = require(__dirname+'/../../db/boards.js') - , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') - , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); +const { buildHomepage } = require(__dirname+'/../../build.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { - //get a list of boards - let boards; try { - boards = await Boards.find(); - await writePageHTML('index.html', 'home.pug', { boards }); + await buildHomepage(); } catch (err) { return next(err); } diff --git a/models/pages/login.js b/models/pages/login.js index e9aadbfa..5fe0c372 100644 --- a/models/pages/login.js +++ b/models/pages/login.js @@ -1,12 +1,12 @@ 'use strict'; -const writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js') +const { buildLogin } = require(__dirname+'/../../build.js') , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { try { - await writePageHTML('login.html', 'login.pug'); + await buildLogin(); } catch (err) { return next(err); } diff --git a/models/pages/manage.js b/models/pages/manage.js index 7319f3da..76225c3d 100644 --- a/models/pages/manage.js +++ b/models/pages/manage.js @@ -2,7 +2,6 @@ const Posts = require(__dirname+'/../../db/posts.js') , Bans = require(__dirname+'/../../db/bans.js') - , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); module.exports = async (req, res, next) => { diff --git a/models/pages/register.js b/models/pages/register.js index 58b51f1b..caa5eece 100644 --- a/models/pages/register.js +++ b/models/pages/register.js @@ -1,12 +1,12 @@ 'use strict'; -const writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js') +const { buildRegister } = require(__dirname+'/../../build.js') , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { try { - await writePageHTML('register.html', 'register.pug'); + await buildRegister(); } catch (err) { return next(err); } diff --git a/models/pages/thread.js b/models/pages/thread.js index 95ad9570..16c20c5f 100644 --- a/models/pages/thread.js +++ b/models/pages/thread.js @@ -1,28 +1,16 @@ 'use strict'; -const Posts = require(__dirname+'/../../db/posts.js') - , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') - , writePageHTML = require(__dirname+'/../../helpers/writepagehtml.js'); +const { buildThread } = require(__dirname+'/../../build.js') + , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js'); module.exports = async (req, res, next) => { - //get the recently bumped thread & preview posts - let thread; - let threadURL; - try { - thread = await Posts.getThread(req.params.board, req.params.id); - if (!thread) { - return res.status(404).render('404'); - } - threadURL = `${req.params.board}/thread/${req.params.id}.html`; - await writePageHTML(threadURL, 'thread.pug', { - board: res.locals.board, - thread - }); - } catch (err) { - return next(err); + try { + await buildThread(res.locals.thread.postId, res.locals.board); + } catch (err) { + return next(err); } - return res.sendFile(`${uploadDirectory}html/${threadURL}`); + return res.sendFile(`${uploadDirectory}html/${req.params.board}/thread/${req.params.id}.html`); } diff --git a/views/includes/pages.pug b/views/includes/pages.pug index 09ed2352..a1194c78 100644 --- a/views/includes/pages.pug +++ b/views/includes/pages.pug @@ -7,7 +7,7 @@ else span a(href=`/${board._id}/index.html`) #{1} | -- for(let i = 2; i <= pages; i++) +- for(let i = 2; i <= maxPage; i++) if i === page span a(href=`/${board._id}/${i}.html`) [#{i}] From cca0f4264cb588ce446864485acd91ed7372d884 Mon Sep 17 00:00:00 2001 From: fatchan Date: Sun, 12 May 2019 13:10:02 +0000 Subject: [PATCH 07/27] more guild settings, e.g. force OP file, message, subject, min message length --- controllers/forms.js | 16 +++++++++++++--- views/pages/manage.pug | 15 +++++++++++++++ wipe.js | 12 ++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/controllers/forms.js b/controllers/forms.js index 48f56642..e5875a27 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -174,15 +174,25 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verif if (!req.body.message && numFiles === 0) { errors.push('Must provide a message or file'); } - if (req.body.message && req.body.message.length > 2000) { - errors.push('Message must be 2000 characters or less'); + if (!req.body.thread && (res.locals.board.settings.forceOPFile && res.locals.board.settings.maxFiles > 0)) { + errors.push('Threads must include a file'); } - if (!req.body.thread && (!req.body.message || req.body.message.length === 0)) { + if (!req.body.thread && res.locals.board.settings.forceOPMessage && (!req.body.message || req.body.message.length === 0)) { errors.push('Threads must include a message'); } + if (req.body.message) { + if (req.body.message.length > 2000) { + errors.push('Message must be 2000 characters or less'); + } else if (req.body.message.length < res.locals.board.settings.minMessageLength) { + errors.push(`Message must be at least ${res.locals.board.settings.minMessageLength} characters long`); + } + } if (req.body.name && req.body.name.length > 50) { errors.push('Name must be 50 characters or less'); } + if (res.locals.board.settings.forceOPSubject && (!req.body.subject || req.body.subject.length === 0)) { + errors.push('Threads must include a subject'); + } if (req.body.subject && req.body.subject.length > 50) { errors.push('Subject must be 50 characters or less'); } diff --git a/views/pages/manage.pug b/views/pages/manage.pug index 235aa902..69e3b9ca 100644 --- a/views/pages/manage.pug +++ b/views/pages/manage.pug @@ -21,9 +21,24 @@ block content label.postform-style.ph-5 input(type='checkbox', name='force_anon', value='true' checked=board.settings.forceAnon) | Disable names and only allow sage email + section.postform-row + .postform-label Force OP Message + label.postform-style.ph-5 + input(type='checkbox', name='force_op_message', value='true' checked=board.settings.forceOPMessage) + section.postform-row + .postform-label Force OP Subject + label.postform-style.ph-5 + input(type='checkbox', name='force_op_subject', value='true' checked=board.settings.forceOPSubject) + section.postform-row + .postform-label Force OP File + label.postform-style.ph-5 + input(type='checkbox', name='force_op_file', value='true' checked=board.settings.forceOPFile) section.postform-row .postform-label Anon Name input(type='text' name='default_name' placeholder=board.settings.defaultName) + section.postform-row + .postform-label Min Message Length + input(type='text' name='min_message_length' placeholder=board.settings.minMessageLength) section.postform-row .postform-label Thread Limit input(type='text' name='thread_limit' placeholder=board.settings.threadLimit) diff --git a/wipe.js b/wipe.js index 817f4e90..18a5896d 100644 --- a/wipe.js +++ b/wipe.js @@ -47,6 +47,10 @@ const Mongo = require(__dirname+'/db/db.js') threadLimit: 100, replyLimit: 300, maxFiles: 3, + forceOPSubject: false, + forceOPMessage: true, + forceOPFile: true, + minMessageLength: 0, defaultName: 'Anonymous', } }) @@ -63,6 +67,10 @@ const Mongo = require(__dirname+'/db/db.js') threadLimit: 100, replyLimit: 300, maxFiles: 3, + forceOPSubject: false, + forceOPMessage: true, + forceOPFile: true, + minMessageLength: 0, defaultName: 'Anonymous', } }) @@ -79,6 +87,10 @@ const Mongo = require(__dirname+'/db/db.js') threadLimit: 100, replyLimit: 300, maxFiles: 0, + forceOPSubject: false, + forceOPMessage: true, + forceOPFile: true, + minMessageLength: 0, defaultName: 'Anonymous', } }) From 39f8ed78c811c0bd0f43bdf1b189d51098f1d373 Mon Sep 17 00:00:00 2001 From: fatchan Date: Sun, 12 May 2019 14:57:02 +0000 Subject: [PATCH 08/27] remove EXIF from images, delete temp files on upload, correct thread pruning to on new thread post, not new reply --- controllers/forms.js | 10 ++++++-- helpers/files/imageupload.js | 19 ++++++++++++++ .../files/{file-upload.js => videoupload.js} | 5 ++-- models/forms/make-post.js | 25 +++++++++++-------- models/forms/uploadbanners.js | 11 +++++--- 5 files changed, 50 insertions(+), 20 deletions(-) create mode 100644 helpers/files/imageupload.js rename helpers/files/{file-upload.js => videoupload.js} (52%) diff --git a/controllers/forms.js b/controllers/forms.js index e5875a27..b6884080 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -8,6 +8,7 @@ const express = require('express') , Trips = require(__dirname+'/../db/trips.js') , Bans = require(__dirname+'/../db/bans.js') , Mongo = require(__dirname+'/../db/db.js') + , remove = require('fs-extra').remove , deletePosts = require(__dirname+'/../models/forms/delete-post.js') , dismissGlobaReports = require(__dirname+'/../models/forms/dismissglobalreport.js') , banPoster = require(__dirname+'/../models/forms/ban-poster.js') @@ -174,7 +175,7 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verif if (!req.body.message && numFiles === 0) { errors.push('Must provide a message or file'); } - if (!req.body.thread && (res.locals.board.settings.forceOPFile && res.locals.board.settings.maxFiles > 0)) { + if (!req.body.thread && (res.locals.board.settings.forceOPFile && res.locals.board.settings.maxFiles === 0)) { errors.push('Threads must include a file'); } if (!req.body.thread && res.locals.board.settings.forceOPMessage && (!req.body.message || req.body.message.length === 0)) { @@ -214,8 +215,13 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verif try { await makePost(req, res, next, numFiles); } catch (err) { + //handler errors here better if (numFiles > 0) { - const fileNames = req.files.file.map(file => file.filename); + const fileNames = [] + for (let i = 0; i < req.files.file.length; i++) { + await remove(req.files.file[i].tempFilePath).catch(e => console.error); + fileNames.push(file.filename); + } await deletePostFiles(fileNames).catch(err => console.error); } return next(err); diff --git a/helpers/files/imageupload.js b/helpers/files/imageupload.js new file mode 100644 index 00000000..310143f4 --- /dev/null +++ b/helpers/files/imageupload.js @@ -0,0 +1,19 @@ +'use strict'; + +const uploadDirectory = require(__dirname+'/../uploadDirectory.js') + , gm = require('@tohru/gm'); + +module.exports = (file, filename, folder) => { + + return new Promise((resolve, reject) => { + gm(file.tempFilePath) + .noProfile() + .write(`${uploadDirectory}${folder}/${filename}`, function (err) { + if (err) { + return reject(err); + } + return resolve(); + }); + }); + +}; diff --git a/helpers/files/file-upload.js b/helpers/files/videoupload.js similarity index 52% rename from helpers/files/file-upload.js rename to helpers/files/videoupload.js index 1bf7953b..a709d272 100644 --- a/helpers/files/file-upload.js +++ b/helpers/files/videoupload.js @@ -1,9 +1,8 @@ 'use strict'; -const configs = require(__dirname+'/../../configs/main.json') - , uploadDirectory = require(__dirname+'/../uploadDirectory.js'); +const uploadDirectory = require(__dirname+'/../uploadDirectory.js'); -module.exports = (req, res, file, filename, folder) => { +module.exports = (file, filename, folder) => { return new Promise((resolve, reject) => { file.mv(`${uploadDirectory}${folder}/${filename}`, function (err) { diff --git a/models/forms/make-post.js b/models/forms/make-post.js index 00991c73..a42989d9 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -21,7 +21,8 @@ const uuidv4 = require('uuid/v4') } , nameRegex = /^(?[^\s#]+)?(?:##(?[^ ]{1}[^\s#]+))?(?:## (?[^\s#]+))?$/ , permsCheck = require(__dirname+'/../../helpers/hasperms.js') - , fileUpload = require(__dirname+'/../../helpers/files/file-upload.js') + , imageUpload = require(__dirname+'/../../helpers/files/imageupload.js') + , videoUpload = require(__dirname+'/../../helpers/files/videoupload.js') , fileCheckMimeType = require(__dirname+'/../../helpers/files/file-check-mime-types.js') , imageThumbnail = require(__dirname+'/../../helpers/files/image-thumbnail.js') , imageIdentify = require(__dirname+'/../../helpers/files/image-identify.js') @@ -91,9 +92,6 @@ module.exports = async (req, res, next, numFiles) => { const filename = uuid + path.extname(file.name); file.filename = filename; //for error to delete failed files - //upload file - await fileUpload(req, res, file, filename, 'img'); - //get metadata let processedFile = { filename: filename, @@ -106,6 +104,7 @@ module.exports = async (req, res, next, numFiles) => { const mainType = file.mimetype.split('/')[0]; switch (mainType) { case 'image': + await imageUpload(file, filename, 'img'); const imageData = await imageIdentify(filename, 'img'); processedFile.geometry = imageData.size // object with width and height pixels processedFile.sizeString = formatSize(processedFile.size) // 123 Ki string @@ -114,6 +113,7 @@ module.exports = async (req, res, next, numFiles) => { break; case 'video': //video metadata + await videoUpload(file, filename, 'img'); const videoData = await videoIdentify(filename); processedFile.duration = videoData.format.duration; processedFile.durationString = new Date(videoData.format.duration*1000).toLocaleString('en-US', {hour12:false}).split(' ')[1].replace(/^00:/, ''); @@ -126,6 +126,9 @@ module.exports = async (req, res, next, numFiles) => { return next(err); } + //delete the temp file + await remove(file.tempFilePath); + //handle gifs with multiple geometry and size if (Array.isArray(processedFile.geometry)) { processedFile.geometry = processedFile.geometry[0]; @@ -137,6 +140,7 @@ module.exports = async (req, res, next, numFiles) => { processedFile.geometryString = processedFile.geometryString[0]; } files.push(processedFile); + } } @@ -230,12 +234,7 @@ module.exports = async (req, res, next, numFiles) => { //always need to rebuild catalog parallelPromises.push(buildCatalog(res.locals.board)); if (data.thread) { - //if we just added a new thread, prune any old ones - const prunedThreads = await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); - for (let i = 0; i < prunedThreads.length; i++) { - parallelPromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`)); - } - //refresh the thread itself + //new reply, so build the thread first parallelPromises.push(buildThread(thread.postId, res.locals.board)); //refersh pages const threadPage = await Posts.getThreadPage(req.params.board, thread); @@ -247,7 +246,11 @@ module.exports = async (req, res, next, numFiles) => { parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPage)); } } else { - //new thread, rebuild all pages + //new thread, rebuild all pages and prune old threads + const prunedThreads = await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); + for (let i = 0; i < prunedThreads.length; i++) { + parallelPromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`)); + } parallelPromises.push(buildBoardMultiple(res.locals.board, 1, 10)); } await Promise.all(parallelPromises); diff --git a/models/forms/uploadbanners.js b/models/forms/uploadbanners.js index b3f40416..64f97378 100644 --- a/models/forms/uploadbanners.js +++ b/models/forms/uploadbanners.js @@ -2,8 +2,9 @@ const uuidv4 = require('uuid/v4') , path = require('path') + , remove = require('fs-extra').remove , uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js') - , fileUpload = require(__dirname+'/../../helpers/files/file-upload.js') + , imageUpload = require(__dirname+'/../../helpers/files/imageupload.js') , fileCheckMimeType = require(__dirname+'/../../helpers/files/file-check-mime-types.js') , deleteFailedFiles = require(__dirname+'/../../helpers/files/deletefailed.js') , imageIdentify = require(__dirname+'/../../helpers/files/image-identify.js') @@ -35,9 +36,10 @@ module.exports = async (req, res, next, numFiles) => { // try to save try { //upload it - await fileUpload(req, res, file, filename, 'banner'); + await imageUpload(file, filename, 'banner'); const imageData = await imageIdentify(filename, 'banner'); const geometry = imageData.size; + await remove(file.tempFilePath); //make sure its 300x100 banner if (geometry.width !== 300 || geometry.height !== 100) { await deleteFailedFiles(filenames, 'banner'); @@ -48,8 +50,9 @@ module.exports = async (req, res, next, numFiles) => { }); } } catch (err) { - //TODO: this better - await deleteFailedFiles(filenames, 'banner'); + //TODO: this better, catch errors some how + await remove(file.tempFilePath).catch(e => console.error); + await deleteFailedFiles(filenames, 'banner').catch(e => console.error); return next(err); } } From 996c616d7ee63c29309212377ea90d0f607fbb34 Mon Sep 17 00:00:00 2001 From: fatchan Date: Mon, 13 May 2019 15:38:22 +0000 Subject: [PATCH 09/27] better handling of banner uploads, failure cases and add checkecbanners to exempt arrays in paramconverter --- controllers/forms.js | 19 ++++++++++++---- helpers/paramconverter.js | 2 +- models/forms/uploadbanners.js | 42 +++++++++++++++++------------------ views/pages/board.pug | 2 +- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/controllers/forms.js b/controllers/forms.js index b6884080..b14ed148 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -26,6 +26,7 @@ const express = require('express') , verifyCaptcha = require(__dirname+'/../helpers/captchaverify.js') , actionHandler = require(__dirname+'/../models/forms/actionhandler.js') , csrf = require(__dirname+'/../helpers/csrfmiddleware.js') + , deleteFailedFiles = require(__dirname+'/../helpers/files/deletefailed.js') , actionChecker = require(__dirname+'/../helpers/actionchecker.js'); @@ -219,10 +220,10 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verif if (numFiles > 0) { const fileNames = [] for (let i = 0; i < req.files.file.length; i++) { - await remove(req.files.file[i].tempFilePath).catch(e => console.error); - fileNames.push(file.filename); + remove(req.files.file[i].tempFilePath).catch(e => console.error); + fileNames.push(req.files.file[i].filename); } - await deletePostFiles(fileNames).catch(err => console.error); + deletePostFiles(fileNames).catch(err => console.error); } return next(err); } @@ -280,6 +281,9 @@ router.post('/board/:board/addbanners', csrf, Boards.exists, checkPermsMiddlewar if (numFiles === 0) { errors.push('Must provide a file'); } + if (res.locals.board.banners.length > 100) { + errors.push('Limit of 100 banners reached'); + } if (errors.length > 0) { return res.status(400).render('message', { @@ -292,7 +296,14 @@ router.post('/board/:board/addbanners', csrf, Boards.exists, checkPermsMiddlewar try { await uploadBanners(req, res, next, numFiles); } catch (err) { - console.error(err); + const fileNames = []; + if (numFiles > 0) { + for (let i = 0; i < req.files.file.length; i++) { + remove(req.files.file[i].tempFilePath).catch(e => console.error); + fileNames.push(req.files.file[i].filename); + } + } + deleteFailedFiles(fileNames, 'banner').catch(e => console.error); return next(err); } diff --git a/helpers/paramconverter.js b/helpers/paramconverter.js index c2315623..6ff9035b 100644 --- a/helpers/paramconverter.js +++ b/helpers/paramconverter.js @@ -1,7 +1,7 @@ 'use strict'; const Mongo = require(__dirname+'/../db/db.js') - , allowedArrays = new Set(['checkedposts', 'globalcheckedposts', 'checkedbans']) + , allowedArrays = new Set(['checkedposts', 'globalcheckedposts', 'checkedbans', 'checkedbanners']) module.exports = (req, res, next) => { diff --git a/models/forms/uploadbanners.js b/models/forms/uploadbanners.js index 64f97378..bbf26f80 100644 --- a/models/forms/uploadbanners.js +++ b/models/forms/uploadbanners.js @@ -31,33 +31,33 @@ module.exports = async (req, res, next, numFiles) => { const file = req.files.file[i]; const uuid = uuidv4(); const filename = uuid + path.extname(file.name); - //add filenames to array add processing to delete previous if one fails + file.filename = filename; //for error to delete failed files filenames.push(filename); - // try to save - try { - //upload it - await imageUpload(file, filename, 'banner'); - const imageData = await imageIdentify(filename, 'banner'); - const geometry = imageData.size; - await remove(file.tempFilePath); - //make sure its 300x100 banner - if (geometry.width !== 300 || geometry.height !== 100) { - await deleteFailedFiles(filenames, 'banner'); - return res.status(400).render('message', { - 'title': 'Bad request', - 'message': `Invalid file ${file.name}. Banners must be 300x100.`, - 'redirect': redirect - }); + + //upload it + await imageUpload(file, filename, 'banner'); + const imageData = await imageIdentify(filename, 'banner'); + const geometry = imageData.size; + await remove(file.tempFilePath); + + //make sure its 300x100 banner + if (geometry.width !== 300 || geometry.height !== 100) { + const fileNames = []; + for (let i = 0; i < req.files.file.length; i++) { + remove(req.files.file[i].tempFilePath).catch(e => console.error); + fileNames.push(req.files.file[i].filename); } - } catch (err) { - //TODO: this better, catch errors some how - await remove(file.tempFilePath).catch(e => console.error); - await deleteFailedFiles(filenames, 'banner').catch(e => console.error); - return next(err); + deleteFailedFiles(fileNames, 'banner').catch(e => console.error); + return res.status(400).render('message', { + 'title': 'Bad request', + 'message': `Invalid file ${file.name}. Banners must be 300x100.`, + 'redirect': redirect + }); } } await Boards.addBanners(req.params.board, filenames); +// await buildBanners(res.locals.board); return res.render('message', { 'title': 'Success', diff --git a/views/pages/board.pug b/views/pages/board.pug index 8aa26cb6..3b225c36 100644 --- a/views/pages/board.pug +++ b/views/pages/board.pug @@ -15,7 +15,7 @@ block content a(href=`/${board._id}/catalog.html`) [Catalog] hr(size=1) form(action='/forms/board/'+board._id+'/actions' method='POST' enctype='application/x-www-form-urlencoded') - input(type='hidden' name='_csrf' value=csrf) + //input(type='hidden' name='_csrf' value=csrf) if threads.length === 0 p No posts. hr(size=1) From a5c120dfd7f4a00657c6f7f4b146a59176aa7698 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 14 May 2019 18:19:04 +0000 Subject: [PATCH 10/27] show banner rotate always (need to add default banner) --- views/includes/boardheader.pug | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/views/includes/boardheader.pug b/views/includes/boardheader.pug index a2daad93..24deff11 100644 --- a/views/includes/boardheader.pug +++ b/views/includes/boardheader.pug @@ -1,6 +1,5 @@ section.board-header - if board.banners.length > 0 - object.board-banner(data=`/banners?board=${board._id}` width='300' height='100') + object.board-banner(data=`/banners?board=${board._id}` width='300' height='100') a.no-decoration(href=`/${board._id}/index.html`) h1.board-title /#{board._id}/ - #{board.name} h4.board-description #{board.description} From f7a5ce50ddc8c8d9be1d1063f0cd06293d1049f4 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 14 May 2019 18:57:35 +0000 Subject: [PATCH 11/27] samesite, secure and htponly for cookies --- server.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server.js b/server.js index 5fca151e..0720a0a2 100644 --- a/server.js +++ b/server.js @@ -37,11 +37,17 @@ const express = require('express') })); // session store + app.set('trust proxy', 1); app.use(session({ secret: configs.sessionSecret, store: new MongoStore({ db: Mongo.client.db('sessions') }), resave: false, - saveUninitialized: false + saveUninitialized: false, + cookie: { + httpOnly: true, + secure: true, + sameSite: 'lax', + } })); app.use(cookieParser()); @@ -89,7 +95,7 @@ const express = require('express') }) // listen - app.listen(configs.port, () => { + app.listen(configs.port, '127.0.0.1', () => { console.log(`Listening on port ${configs.port}`); }); From 54ff8b8c0e5ecb8d240ea47501a6a08901e4d8c9 Mon Sep 17 00:00:00 2001 From: fatchan Date: Wed, 15 May 2019 07:05:55 +0000 Subject: [PATCH 12/27] details/summary tags instead of hacky hidden post form check css --- gulp/res/css/style.css | 7 +- views/includes/actionfooter.pug | 113 +++++++++---------- views/includes/actionfooter_globalmanage.pug | 62 +++++----- views/includes/actionfooter_manage.pug | 69 ++++++----- views/includes/postform.pug | 82 +++++++------- views/pages/board.pug | 2 +- views/pages/thread.pug | 1 - 7 files changed, 168 insertions(+), 168 deletions(-) diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index 115e36ef..b1c42aea 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -72,6 +72,7 @@ object { .navbar { border-bottom: 1px solid #a9a9a9; + background: #d6daf0; } .catalog-tile-button { @@ -133,6 +134,7 @@ object { } .mode { + margin-top: 1px; background-color: red; color: white; font-weight: bold; @@ -211,7 +213,8 @@ td, th { } .actions { - max-width: 100%; + text-align: left; + max-width: 200px; display: flex; flex-direction: column; margin: 2px 0; @@ -439,6 +442,7 @@ input textarea { margin-top: auto; line-height: 30px; border-top: 1px solid #a9a9a9; + background: #d6daf0; } input[type="text"], input[type="submit"], input[type="password"], input[type="file"], textarea { @@ -536,6 +540,7 @@ hr { .catalog-tile { overflow-y: hidden; + width: 48%; } .table-body { diff --git a/views/includes/actionfooter.pug b/views/includes/actionfooter.pug index 1a1bf0ea..f8255dd5 100644 --- a/views/includes/actionfooter.pug +++ b/views/includes/actionfooter.pug @@ -1,57 +1,56 @@ -label.toggle-label Toggle Post Actions - input.toggle(type='checkbox') - .action-wrapper.togglable - .actions - h4.no-m-p Actions: - label - input.post-check(type='checkbox', name='delete' value=1) - | Delete - label - input.post-check(type='checkbox', name='delete_file' value=1) - | Delete File Only - label - input.post-check(type='checkbox', name='spoiler' value=1) - | Spoiler Images - label - input#password(type='text', name='password', placeholder='post password' autocomplete='off') - label - input.post-check(type='checkbox', name='report' value=1) - | Report - label - input.post-check(type='checkbox', name='global_report' value=1) - | Global Report - label - input#report(type='text', name='report_reason', placeholder='report reason' autocomplete='off') - .actions - h4.no-m-p Mod Actions: - label - input.post-check(type='checkbox', name='delete_ip_board' value=1) - | Delete from IP on board - label - input.post-check(type='checkbox', name='delete_ip_global' value=1) - | Delete from IP globally - label - input.post-check(type='checkbox', name='sticky' value=1) - | Sticky - label - input.post-check(type='checkbox', name='lock' value=1) - | Lock - label - input.post-check(type='checkbox', name='sage' value=1) - | Sage - label - input.post-check(type='checkbox', name='ban' value=1) - | Ban Poster - label - input.post-check(type='checkbox', name='global_ban' value=1) - | Global Ban Poster - label - input.post-check(type='checkbox', name='preserve_post' value=1) - | Show Post In Ban - label - input#report(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') - .actions - h4.no-m-p Captcha: - img.captcha(src='/captcha' width=200 height=80) - input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') - input(type='submit', value='submit') +details.toggle-label + summary Show Post Actions + .actions + h4.no-m-p Actions: + label + input.post-check(type='checkbox', name='delete' value=1) + | Delete + label + input.post-check(type='checkbox', name='delete_file' value=1) + | Delete File Only + label + input.post-check(type='checkbox', name='spoiler' value=1) + | Spoiler Images + label + input#password(type='text', name='password', placeholder='post password' autocomplete='off') + label + input.post-check(type='checkbox', name='report' value=1) + | Report + label + input.post-check(type='checkbox', name='global_report' value=1) + | Global Report + label + input#report(type='text', name='report_reason', placeholder='report reason' autocomplete='off') + .actions + h4.no-m-p Mod Actions: + label + input.post-check(type='checkbox', name='delete_ip_board' value=1) + | Delete from IP on board + label + input.post-check(type='checkbox', name='delete_ip_global' value=1) + | Delete from IP globally + label + input.post-check(type='checkbox', name='sticky' value=1) + | Sticky + label + input.post-check(type='checkbox', name='lock' value=1) + | Lock + label + input.post-check(type='checkbox', name='sage' value=1) + | Sage + label + input.post-check(type='checkbox', name='ban' value=1) + | Ban Poster + label + input.post-check(type='checkbox', name='global_ban' value=1) + | Global Ban Poster + label + input.post-check(type='checkbox', name='preserve_post' value=1) + | Show Post In Ban + label + input#report(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') + .actions + h4.no-m-p Captcha: + img.captcha(src='/captcha' width=200 height=80) + input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') + input(type='submit', value='submit') diff --git a/views/includes/actionfooter_globalmanage.pug b/views/includes/actionfooter_globalmanage.pug index d91014e8..7a18be5c 100644 --- a/views/includes/actionfooter_globalmanage.pug +++ b/views/includes/actionfooter_globalmanage.pug @@ -1,32 +1,30 @@ -label.toggle-label Toggle Post Actions - input.toggle(type='checkbox') - .action-wrapper.togglable - .actions - h4.no-m-p Actions: - label - input.post-check(type='checkbox', name='delete' value=1) - | Delete - label - input.post-check(type='checkbox', name='delete_file' value=1) - | Delete File Only - label - input.post-check(type='checkbox', name='spoiler' value=1) - | Spoiler Images - label - input#report(type='text', name='report_reason', placeholder='report reason' autocomplete='off') - label - input.post-check(type='checkbox', name='delete_ip_global' value=1) - | Delete from IP globally - label - input.post-check(type='checkbox', name='global_dismiss' value=1) - | Dismiss Global Reports - label - input.post-check(type='checkbox', name='global_ban' value=1) - | Global Ban Poster - label - input.post-check(type='checkbox', name='preserve_post' value=1) - | Show Post In Ban - label - input#ban_reason(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') - input(type='submit', value='submit') - +details.toggle-label + summary Show Post Actions + .actions + h4.no-m-p Actions: + label + input.post-check(type='checkbox', name='delete' value=1) + | Delete + label + input.post-check(type='checkbox', name='delete_file' value=1) + | Delete File Only + label + input.post-check(type='checkbox', name='spoiler' value=1) + | Spoiler Images + label + input#report(type='text', name='report_reason', placeholder='report reason' autocomplete='off') + label + input.post-check(type='checkbox', name='delete_ip_global' value=1) + | Delete from IP globally + label + input.post-check(type='checkbox', name='global_dismiss' value=1) + | Dismiss Global Reports + label + input.post-check(type='checkbox', name='global_ban' value=1) + | Global Ban Poster + label + input.post-check(type='checkbox', name='preserve_post' value=1) + | Show Post In Ban + label + input#ban_reason(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') + input(type='submit', value='submit') diff --git a/views/includes/actionfooter_manage.pug b/views/includes/actionfooter_manage.pug index aeb1aeb4..5ce46915 100644 --- a/views/includes/actionfooter_manage.pug +++ b/views/includes/actionfooter_manage.pug @@ -1,36 +1,35 @@ -label.toggle-label Toggle Post Actions - input.toggle(type='checkbox') - .action-wrapper.togglable - .actions - h4.no-m-p Actions: - label - input.post-check(type='checkbox', name='delete' value=1) - | Delete - label - input.post-check(type='checkbox', name='delete_file' value=1) - | Delete File Only - label - input.post-check(type='checkbox', name='spoiler' value=1) - | Spoiler Images - label - input.post-check(type='checkbox', name='delete_ip_board' value=1) - | Delete from IP on board - label - input.post-check(type='checkbox', name='delete_ip_global' value=1) - | Delete from IP globally - label - input.post-check(type='checkbox', name='dismiss' value=1) - | Dismiss Reports - label - input.post-check(type='checkbox', name='ban' value=1) - | Ban Poster - label - input.post-check(type='checkbox', name='global_ban' value=1) - | Global Ban Poster - label - input.post-check(type='checkbox', name='preserve_post' value=1) - | Show Post In Ban - label - input#ban_reason(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') - input(type='submit', value='submit') +details.toggle-label + summary Show Post Actions + .actions + h4.no-m-p Actions: + label + input.post-check(type='checkbox', name='delete' value=1) + | Delete + label + input.post-check(type='checkbox', name='delete_file' value=1) + | Delete File Only + label + input.post-check(type='checkbox', name='spoiler' value=1) + | Spoiler Images + label + input.post-check(type='checkbox', name='delete_ip_board' value=1) + | Delete from IP on board + label + input.post-check(type='checkbox', name='delete_ip_global' value=1) + | Delete from IP globally + label + input.post-check(type='checkbox', name='dismiss' value=1) + | Dismiss Reports + label + input.post-check(type='checkbox', name='ban' value=1) + | Ban Poster + label + input.post-check(type='checkbox', name='global_ban' value=1) + | Global Ban Poster + label + input.post-check(type='checkbox', name='preserve_post' value=1) + | Show Post In Ban + label + input#ban_reason(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') + input(type='submit', value='submit') diff --git a/views/includes/postform.pug b/views/includes/postform.pug index e34ef2e7..356abf61 100644 --- a/views/includes/postform.pug +++ b/views/includes/postform.pug @@ -1,46 +1,46 @@ section.form-wrapper.flex-center.mv-10 - form.form-post(action=`/forms/board/${board._id}/post`, enctype='multipart/form-data', method='POST') - //input(type='hidden' name='_csrf' value=csrf) - input(type='hidden' name='thread' value=thread != null ? thread.postId : null) - unless board.settings.forceAnon - section.postform-row - .postform-label Name - input#name(type='text', name='name', placeholder=board.defaultName autocomplete='off' maxlength='50') - section.postform-row - .postform-label Subject - input#title(type='text', name='subject', autocomplete='off' maxlength='50') - section.postform-row - .postform-label Email - input#name(type='text', name='email', autocomplete='off' maxlength='50') - else - section.postform-row - .postform-label Sage - label.postform-style.ph-5 - input#spoiler(type='checkbox', name='email', value='sage') - | Sage - if !thread + details.toggle-label + summary Show Post Form + form.form-post(action=`/forms/board/${board._id}/post`, enctype='multipart/form-data', method='POST') + //input(type='hidden' name='_csrf' value=csrf) + input(type='hidden' name='thread' value=thread != null ? thread.postId : null) + unless board.settings.forceAnon + section.postform-row + .postform-label Name + input#name(type='text', name='name', placeholder=board.defaultName autocomplete='off' maxlength='50') section.postform-row .postform-label Subject input#title(type='text', name='subject', autocomplete='off' maxlength='50') - section.postform-row - .postform-label Message - textarea#message(name='message', rows='5', autocomplete='off' maxlength='2000') - if board.settings.maxFiles !== 0 + section.postform-row + .postform-label Email + input#name(type='text', name='email', autocomplete='off' maxlength='50') + else + section.postform-row + .postform-label Sage + label.postform-style.ph-5 + input#spoiler(type='checkbox', name='email', value='sage') + | Sage + if !thread + section.postform-row + .postform-label Subject + input#title(type='text', name='subject', autocomplete='off' maxlength='50') + section.postform-row + .postform-label Message + textarea#message(name='message', rows='5', autocomplete='off' maxlength='2000') + if board.settings.maxFiles !== 0 + section.postform-row + .postform-label Files + input#file(type='file', name='file' multiple='multiple') + label.postform-style.ph-5.ml-1 + input#spoiler(type='checkbox', name='spoiler', value='true') + | Spoiler + section.postform-row + .postform-label Password + input#password(type='password', name='password', autocomplete='off' placeholder='password for deleting post later' maxlength='50') section.postform-row - .postform-label Files - input#file(type='file', name='file' multiple='multiple') - label.postform-style.ph-5.ml-1 - input#spoiler(type='checkbox', name='spoiler', value='true') - | Spoiler - section.postform-row - .postform-label Password - input#password(type='password', name='password', autocomplete='off' placeholder='password for deleting post later' maxlength='50') - section.postform-row - .postform-label Captcha - .postform-col - img.captcha(src='/captcha' width=200 height=80) - input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') - if !thread - input(type='submit', value='New Thread') - else - input(type='submit', value='Reply') + .postform-label Captcha + .postform-col + img.captcha(src='/captcha' width=200 height=80) + input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') + input(type='submit', value=`New ${threads ? 'Thread' : 'Reply'}`) + //.mode Posting mode: #{threads ? 'Thread' : 'Reply'} diff --git a/views/pages/board.pug b/views/pages/board.pug index 3b225c36..4a5ac5e3 100644 --- a/views/pages/board.pug +++ b/views/pages/board.pug @@ -7,7 +7,6 @@ block head block content include ../includes/boardheader.pug include ../includes/postform.pug - .mode Posting mode: Thread nav.pages#top include ../includes/pages.pug a(href='#bottom') [Bottom] @@ -26,6 +25,7 @@ block content +post(post, true) hr(size=1) nav.pages#bottom + include ../includes/pages.pug a(href='#top') [Top] | a(href=`/${board._id}/catalog.html`) [Catalog] diff --git a/views/pages/thread.pug b/views/pages/thread.pug index 8a773247..8854d2fd 100644 --- a/views/pages/thread.pug +++ b/views/pages/thread.pug @@ -12,7 +12,6 @@ block head block content include ../includes/boardheader.pug include ../includes/postform.pug - .mode Posting mode: Reply nav.pages#top a(href='#bottom') [Bottom] | From 9aff8a60312f036d3559b3519f3b8a4fad93236d Mon Sep 17 00:00:00 2001 From: fatchan Date: Thu, 16 May 2019 10:09:38 +0000 Subject: [PATCH 13/27] cache compiled function for templates --- helpers/render.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/render.js b/helpers/render.js index 3708a102..1c3ec885 100644 --- a/helpers/render.js +++ b/helpers/render.js @@ -7,6 +7,6 @@ const outputFile = require('fs-extra').outputFile , templateDirectory = path.join(__dirname+'/../views/pages/'); module.exports = async (htmlName, templateName, options) => { - const html = pug.renderFile(`${templateDirectory}${templateName}`, options); + const html = pug.renderFile(`${templateDirectory}${templateName}`, { ...options, cache: true }); return outputFile(`${uploadDirectory}html/${htmlName}`, html); }; From 205b92275a7ffcbd8aa4c6114dd65a6b2e517c9a Mon Sep 17 00:00:00 2001 From: fatchan Date: Thu, 16 May 2019 11:00:51 +0000 Subject: [PATCH 14/27] more style changes & fixes --- views/includes/boardheader.pug | 1 + views/includes/pages.pug | 21 ++++++++------------- views/includes/postform.pug | 2 +- views/pages/board.pug | 5 ++++- views/pages/catalog.pug | 1 + views/pages/manage.pug | 1 + views/pages/thread.pug | 3 +++ 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/views/includes/boardheader.pug b/views/includes/boardheader.pug index 24deff11..5426d559 100644 --- a/views/includes/boardheader.pug +++ b/views/includes/boardheader.pug @@ -1,5 +1,6 @@ section.board-header object.board-banner(data=`/banners?board=${board._id}` width='300' height='100') + br a.no-decoration(href=`/${board._id}/index.html`) h1.board-title /#{board._id}/ - #{board.name} h4.board-description #{board.description} diff --git a/views/includes/pages.pug b/views/includes/pages.pug index a1194c78..d9456ea4 100644 --- a/views/includes/pages.pug +++ b/views/includes/pages.pug @@ -1,20 +1,15 @@ | Page: if page === 1 - span - a(href=`/${board._id}/index.html`) [#{1}] - | + a(href=`/${board._id}/index.html`) [#{1}] + | else - span - a(href=`/${board._id}/index.html`) #{1} - | + a(href=`/${board._id}/index.html`) #{1} + | - for(let i = 2; i <= maxPage; i++) if i === page - span - a(href=`/${board._id}/${i}.html`) [#{i}] - | - + a(href=`/${board._id}/${i}.html`) [#{i}] + | else - span - a(href=`/${board._id}/${i}.html`) #{i} - | + a(href=`/${board._id}/${i}.html`) #{i} + | | | diff --git a/views/includes/postform.pug b/views/includes/postform.pug index 356abf61..8e649579 100644 --- a/views/includes/postform.pug +++ b/views/includes/postform.pug @@ -1,4 +1,4 @@ -section.form-wrapper.flex-center.mv-10 +section.form-wrapper.flex-center details.toggle-label summary Show Post Form form.form-post(action=`/forms/board/${board._id}/post`, enctype='multipart/form-data', method='POST') diff --git a/views/pages/board.pug b/views/pages/board.pug index 4a5ac5e3..5615b93d 100644 --- a/views/pages/board.pug +++ b/views/pages/board.pug @@ -6,15 +6,17 @@ block head block content include ../includes/boardheader.pug + br include ../includes/postform.pug + br nav.pages#top include ../includes/pages.pug a(href='#bottom') [Bottom] | a(href=`/${board._id}/catalog.html`) [Catalog] + | hr(size=1) form(action='/forms/board/'+board._id+'/actions' method='POST' enctype='application/x-www-form-urlencoded') - //input(type='hidden' name='_csrf' value=csrf) if threads.length === 0 p No posts. hr(size=1) @@ -29,4 +31,5 @@ block content a(href='#top') [Top] | a(href=`/${board._id}/catalog.html`) [Catalog] + br include ../includes/actionfooter.pug diff --git a/views/pages/catalog.pug b/views/pages/catalog.pug index e8ed7379..945732bb 100644 --- a/views/pages/catalog.pug +++ b/views/pages/catalog.pug @@ -6,6 +6,7 @@ block head block content include ../includes/boardheader.pug + br nav.pages#top a(href='#bottom') [Bottom] | diff --git a/views/pages/manage.pug b/views/pages/manage.pug index 69e3b9ca..c6411dad 100644 --- a/views/pages/manage.pug +++ b/views/pages/manage.pug @@ -7,6 +7,7 @@ block head block content include ../includes/boardheader.pug + br h4.no-m-p Settings: section.form-wrapper.flexleft.mv-10 form.form-post(action=`/forms/board/${board._id}/settings` method='POST' enctype='application/x-www-form-urlencoded') diff --git a/views/pages/thread.pug b/views/pages/thread.pug index 8854d2fd..035c26c3 100644 --- a/views/pages/thread.pug +++ b/views/pages/thread.pug @@ -11,7 +11,9 @@ block head block content include ../includes/boardheader.pug + br include ../includes/postform.pug + br nav.pages#top a(href='#bottom') [Bottom] | @@ -32,4 +34,5 @@ block content a(href=`/${board._id}/index.html`) [Return] | a(href=`/${board._id}/catalog.html`) [Catalog] + br include ../includes/actionfooter.pug From 05977d3cd6ee89672a31f5dd109d2ee166603294 Mon Sep 17 00:00:00 2001 From: fatchan Date: Thu, 16 May 2019 11:01:17 +0000 Subject: [PATCH 15/27] server graceful reload with PM2 and close on sigints --- ecosystem.config.js | 3 +++ gulp/res/css/style.css | 21 +++++++++++++++------ server.js | 27 ++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/ecosystem.config.js b/ecosystem.config.js index 06764043..4f17bc93 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -8,6 +8,9 @@ module.exports = { watch: false, max_memory_restart: '1G', log_date_format: 'YYYY-MM-DD HH:mm:ss', + wait_ready: true, + listen_timeout: 10000, + kill_timeout: 10000, env: { NODE_ENV: 'development' }, diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index b1c42aea..6a13f30b 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -48,9 +48,11 @@ body { } .pages { - margin: 10px 0; + box-sizing: border-box; padding: 10px; + width: max-content; } + a, a:visited { text-decoration: underline; color: #34345C; @@ -207,7 +209,7 @@ td, th { align-items: center; } -.post-container, .pages, .toggle-label { +.post-container, .pages, summary { background: #D6DAF0; border: 1px solid #B7C5D9; } @@ -235,8 +237,12 @@ td, th { box-shadow: inset 0 0 100px 100px rgba(255,255,255,.25); } -.toggle-label { +summary { + margin-bottom: 1px; padding: 10px; +} + +.toggle-label { text-align: center; max-width: 100%; box-sizing: border-box; @@ -301,9 +307,8 @@ td, th { text-align: center; margin: 2px; margin-top: 0px; - max-width: 160px; + max-width: 128px; overflow: hidden; - max-width: 160px; text-overflow: ellipsis; word-break: keep-all; } @@ -413,7 +418,7 @@ input textarea { } .nav-item { - line-height: 30px; + line-height: 38px; text-decoration: none; float: left; padding-left: 10px; @@ -534,6 +539,10 @@ hr { height: 8px; } + .pages { + width:100%; + } + .post-container { width: 100%; } diff --git a/server.js b/server.js index 0720a0a2..6d056fd8 100644 --- a/server.js +++ b/server.js @@ -95,8 +95,33 @@ const express = require('express') }) // listen - app.listen(configs.port, '127.0.0.1', () => { + const server = app.listen(configs.port, '127.0.0.1', () => { console.log(`Listening on port ${configs.port}`); }); + //let PM@ know that this is ready (forgraceful reloads) + process.send('ready'); + + process.on('SIGINT', () => { + + console.info('SIGINT signal received.') + + // Stops the server from accepting new connections and finishes existing connections. + server.close((err) => { + + // if error, log and exit with error (1 code) + if (err) { + console.error(err); + process.exit(1); + } + + // close database connection + Mongo.client.close(); + + // now close without error + process.exit(0); + + }) + }) + })(); From bf2c2c9fa122ad957141daade418aa408253014e Mon Sep 17 00:00:00 2001 From: fatchan Date: Sat, 18 May 2019 08:42:21 +0000 Subject: [PATCH 16/27] fix global dismissing, and prevent spoilering already spoilered posts --- controllers/forms.js | 11 +++++++++-- models/forms/spoiler-post.js | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/controllers/forms.js b/controllers/forms.js index b14ed148..cf31baa2 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -10,7 +10,8 @@ const express = require('express') , Mongo = require(__dirname+'/../db/db.js') , remove = require('fs-extra').remove , deletePosts = require(__dirname+'/../models/forms/delete-post.js') - , dismissGlobaReports = require(__dirname+'/../models/forms/dismissglobalreport.js') + , spoilerPosts = require(__dirname+'/../models/forms/spoiler-post.js') + , dismissGlobalReports = require(__dirname+'/../models/forms/dismissglobalreport.js') , banPoster = require(__dirname+'/../models/forms/ban-poster.js') , removeBans = require(__dirname+'/../models/forms/removebans.js') , makePost = require(__dirname+'/../models/forms/make-post.js') @@ -469,7 +470,13 @@ router.post('/global/actions', csrf, checkPermsMiddleware, paramConverter, async combinedQuery[action] = { ...combinedQuery[action], ...query} } messages.push(message); - } + } else if (req.body.spoiler) { + const { message, action, query } = spoilerPosts(posts); + if (action) { + combinedQuery[action] = { ...combinedQuery[action], ...query} + } + messages.push(message); + } if (req.body.global_dismiss) { const { message, action, query } = dismissGlobalReports(posts); if (action) { diff --git a/models/forms/spoiler-post.js b/models/forms/spoiler-post.js index f974b492..121816a3 100644 --- a/models/forms/spoiler-post.js +++ b/models/forms/spoiler-post.js @@ -4,12 +4,12 @@ module.exports = (posts) => { // filter to ones not spoilered const filteredPosts = posts.filter(post => { - return !post.spoiler + return !post.spoiler && post.files.length > 0; }); if (filteredPosts.length === 0) { return { - message:'Post(s) already spoilered' + message:'No post(s) to spoiler' }; } From 27afcafa474f636c46dc3b579a47177bf2f2f1ab Mon Sep 17 00:00:00 2001 From: fatchan Date: Sat, 18 May 2019 08:42:45 +0000 Subject: [PATCH 17/27] showomitted count correctly for threads on baord pages --- db/posts.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/posts.js b/db/posts.js index 3744cd24..dbd727ed 100644 --- a/db/posts.js +++ b/db/posts.js @@ -58,8 +58,8 @@ module.exports = { //reverse order for board page thread.replies = replies.reverse(); - //temporary mitigation for deletion issue - if (replies.length > 5) { + //if enough replies, show omitted count + if (thread.replyposts > 5) { //cout omitted image and posts const numPreviewImages = replies.reduce((acc, post) => { return acc + post.files.length }, 0); thread.omittedimages = thread.replyfiles - numPreviewImages; From f43ed12a8419a2a0df57c8126cf6ecddcb7911e0 Mon Sep 17 00:00:00 2001 From: fatchan Date: Sat, 18 May 2019 08:43:07 +0000 Subject: [PATCH 18/27] change action footer options on manage and global manage pages --- views/includes/actionfooter_globalmanage.pug | 2 -- views/includes/actionfooter_manage.pug | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/views/includes/actionfooter_globalmanage.pug b/views/includes/actionfooter_globalmanage.pug index 7a18be5c..c289a50e 100644 --- a/views/includes/actionfooter_globalmanage.pug +++ b/views/includes/actionfooter_globalmanage.pug @@ -11,8 +11,6 @@ details.toggle-label label input.post-check(type='checkbox', name='spoiler' value=1) | Spoiler Images - label - input#report(type='text', name='report_reason', placeholder='report reason' autocomplete='off') label input.post-check(type='checkbox', name='delete_ip_global' value=1) | Delete from IP globally diff --git a/views/includes/actionfooter_manage.pug b/views/includes/actionfooter_manage.pug index 5ce46915..68163047 100644 --- a/views/includes/actionfooter_manage.pug +++ b/views/includes/actionfooter_manage.pug @@ -11,6 +11,11 @@ details.toggle-label label input.post-check(type='checkbox', name='spoiler' value=1) | Spoiler Images + label + input.post-check(type='checkbox', name='global_report' value=1) + | Global Report + label + input#report(type='text', name='report_reason', placeholder='report reason' autocomplete='off') label input.post-check(type='checkbox', name='delete_ip_board' value=1) | Delete from IP on board @@ -32,4 +37,3 @@ details.toggle-label label input#ban_reason(type='text', name='ban_reason', placeholder='ban reason' autocomplete='off') input(type='submit', value='submit') - From 061188414c43b334f0f81c3671d8f0af9f0bea09 Mon Sep 17 00:00:00 2001 From: fatchan Date: Sat, 18 May 2019 08:43:16 +0000 Subject: [PATCH 19/27] stly fixes --- gulp/res/css/style.css | 1 + 1 file changed, 1 insertion(+) diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index 6a13f30b..8018605a 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -240,6 +240,7 @@ td, th { summary { margin-bottom: 1px; padding: 10px; + cursor: pointer; } .toggle-label { From 67c7cf3650ec2ae3d68840d1ff392d2ffd85a271 Mon Sep 17 00:00:00 2001 From: fatchan Date: Sat, 18 May 2019 08:50:23 +0000 Subject: [PATCH 20/27] check if its a child process before sending pm2 ready signal --- server.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server.js b/server.js index 6d056fd8..7499dc9c 100644 --- a/server.js +++ b/server.js @@ -99,8 +99,10 @@ const express = require('express') console.log(`Listening on port ${configs.port}`); }); - //let PM@ know that this is ready (forgraceful reloads) - process.send('ready'); + //let PM2 know that this is ready (for graceful reloads) + if (typeof process.send === 'function') { //make sure we are a child process + process.send('ready'); + } process.on('SIGINT', () => { From efa57023e7bd35bef6e3c04d2258ab66e6032872 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 10:53:01 +0000 Subject: [PATCH 21/27] board settings allow to toggle captchas and check to captchaverify --- helpers/captchaverify.js | 4 ++++ views/includes/postform.pug | 11 ++++++----- views/pages/manage.pug | 4 ++++ wipe.js | 24 ++++-------------------- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/helpers/captchaverify.js b/helpers/captchaverify.js index 8aca1508..eec25ccd 100644 --- a/helpers/captchaverify.js +++ b/helpers/captchaverify.js @@ -7,6 +7,10 @@ const Captchas = require(__dirname+'/../db/captchas.js') module.exports = async (req, res, next) => { + if (!res.locals.board.settings.captcha) { + return next(); + } + //check if captcha field in form is valid const input = req.body.captcha; if (!input || input.length !== 6) { diff --git a/views/includes/postform.pug b/views/includes/postform.pug index 8e649579..0dac0410 100644 --- a/views/includes/postform.pug +++ b/views/includes/postform.pug @@ -37,10 +37,11 @@ section.form-wrapper.flex-center section.postform-row .postform-label Password input#password(type='password', name='password', autocomplete='off' placeholder='password for deleting post later' maxlength='50') - section.postform-row - .postform-label Captcha - .postform-col - img.captcha(src='/captcha' width=200 height=80) - input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') + if board.settings.captcha + section.postform-row + .postform-label Captcha + .postform-col + img.captcha(src='/captcha' width=200 height=80) + input#captcha(type='text', name='captcha', autocomplete='off' placeholder='captcha text' maxlength='6') input(type='submit', value=`New ${threads ? 'Thread' : 'Reply'}`) //.mode Posting mode: #{threads ? 'Thread' : 'Reply'} diff --git a/views/pages/manage.pug b/views/pages/manage.pug index c6411dad..43417353 100644 --- a/views/pages/manage.pug +++ b/views/pages/manage.pug @@ -22,6 +22,10 @@ block content label.postform-style.ph-5 input(type='checkbox', name='force_anon', value='true' checked=board.settings.forceAnon) | Disable names and only allow sage email + section.postform-row + .postform-label Post Captcha + label.postform-style.ph-5 + input(type='checkbox', name='captcha', value='true' checked=board.settings.captcha) section.postform-row .postform-label Force OP Message label.postform-style.ph-5 diff --git a/wipe.js b/wipe.js index 18a5896d..e35fe808 100644 --- a/wipe.js +++ b/wipe.js @@ -42,6 +42,7 @@ const Mongo = require(__dirname+'/db/db.js') moderators: [], banners: [], settings: { + captcha: false, forceAnon: true, ids: true, threadLimit: 100, @@ -62,6 +63,7 @@ const Mongo = require(__dirname+'/db/db.js') moderators: [], banners: [], settings: { + captcha: true, forceAnon: false, ids: false, threadLimit: 100, @@ -82,6 +84,7 @@ const Mongo = require(__dirname+'/db/db.js') moderators: [], banners: [], settings: { + captcha: true, forceAnon: true, ids: false, threadLimit: 100, @@ -129,28 +132,9 @@ const Mongo = require(__dirname+'/db/db.js') } } }); - await readdir('static/img/').then(async files => { - await Promise.all(files.map(async file => { - unlink(path.join('static/img/', file)); - })) - }); - await readdir('static/captcha/').then(async files => { - await Promise.all(files.map(async file => { - unlink(path.join('static/captcha/', file)); - })) - }); - await readdir('static/banner/').then(async files => { - await Promise.all(files.map(async file => { - unlink(path.join('static/banner/', file)); - })) - }); - await readdir('static/html/').then(async files => { - await Promise.all(files.map(async file => { - unlink(path.join('static/html/', file)); - })) - }); console.log('creating admin account: admin:changeme'); await Accounts.insertOne('admin', 'changeme', 3); + Mongo.client.close() console.log('done'); process.exit(0); })(); From 38c72dd5ce67ea220f60eb7f80d7f1a511fee853 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 10:53:32 +0000 Subject: [PATCH 22/27] improve ecosystem file and only signal PM2 _after_ listening --- ecosystem.config.js | 4 ++-- server.js | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ecosystem.config.js b/ecosystem.config.js index 4f17bc93..9b6f62b3 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -9,8 +9,8 @@ module.exports = { max_memory_restart: '1G', log_date_format: 'YYYY-MM-DD HH:mm:ss', wait_ready: true, - listen_timeout: 10000, - kill_timeout: 10000, + listen_timeout: 5000, + kill_timeout: 5000, env: { NODE_ENV: 'development' }, diff --git a/server.js b/server.js index 7499dc9c..eb8f1412 100644 --- a/server.js +++ b/server.js @@ -96,13 +96,16 @@ const express = require('express') // listen const server = app.listen(configs.port, '127.0.0.1', () => { + console.log(`Listening on port ${configs.port}`); - }); - //let PM2 know that this is ready (for graceful reloads) - if (typeof process.send === 'function') { //make sure we are a child process - process.send('ready'); - } + //let PM2 know that this is ready (for graceful reloads) + if (typeof process.send === 'function') { //make sure we are a child process + console.info('Sending ready signal to PM2') + process.send('ready'); + } + + }); process.on('SIGINT', () => { From 826b62347707d03041f38f00a2b12baff088c1a1 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 11:07:47 +0000 Subject: [PATCH 23/27] dont thumbnail images that are already small dimensions, and fix css for mismatched width/height thumbs --- gulp/res/css/style.css | 6 ++++++ models/forms/make-post.js | 8 +++++++- views/mixins/post.pug | 10 ++++++---- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index 8018605a..7bfa3365 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -312,12 +312,18 @@ summary { overflow: hidden; text-overflow: ellipsis; word-break: keep-all; + font-size: x-small; } .post-file-src { margin: 0 auto; } +.file-thumb { + max-width: 128px; + max-height: 128px; +} + figure { } diff --git a/models/forms/make-post.js b/models/forms/make-post.js index a42989d9..aa0a1f58 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -109,7 +109,12 @@ module.exports = async (req, res, next, numFiles) => { processedFile.geometry = imageData.size // object with width and height pixels processedFile.sizeString = formatSize(processedFile.size) // 123 Ki string processedFile.geometryString = imageData.Geometry // 123 x 123 string - await imageThumbnail(filename); + if (processedFile.geometry.height <= 128 && processedFile.geometry.width <= 128) { + processedFile.hasThumb = false; + } else { + processedFile.hasThumb = true; + await imageThumbnail(filename); + } break; case 'video': //video metadata @@ -120,6 +125,7 @@ module.exports = async (req, res, next, numFiles) => { processedFile.geometry = {width: videoData.streams[0].coded_width, height: videoData.streams[0].coded_height} // object with width and height pixels processedFile.sizeString = formatSize(processedFile.size) // 123 Ki string processedFile.geometryString = `${processedFile.geometry.width}x${processedFile.geometry.height}` // 123 x 123 string + processedFile.hasThumb = true; await videoThumbnail(filename); break; default: diff --git a/views/mixins/post.pug b/views/mixins/post.pug index 0d9785ee..a8ae0fd3 100644 --- a/views/mixins/post.pug +++ b/views/mixins/post.pug @@ -41,7 +41,7 @@ mixin post(post, truncate, manage=false, globalmanage=false) .post-files each file in post.files .post-file - small.post-file-info + span.post-file-info span: a(href='/img/'+file.filename title=file.originalFilename download=file.originalFilename) #{file.originalFilename} br span @@ -50,11 +50,13 @@ mixin post(post, truncate, manage=false, globalmanage=false) | , #{file.durationString} | ) .post-file-src - a(target='_blank' href='/img/'+file.filename) + a(target='_blank' href=`/img/${file.filename}`) if post.spoiler - object(data='/img/spoiler.png' width='128' height='128') + object.file-thumb(data='/img/spoiler.png' width='128' height='128') + else if file.hasThumb + object.file-thumb(data=`/img/thumb-${file.filename.split('.')[0]}.jpg`) else - object(data=`/img/thumb-${file.filename.split('.')[0]}.jpg`) + object.file-thumb(data=`/img/${file.filename}`) if post.message if truncate - From 6d7d71cbb6752265bb8de799229b3e0828ce05a2 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 11:58:49 +0000 Subject: [PATCH 24/27] fix captcha enabled enforcing --- helpers/captchaverify.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/captchaverify.js b/helpers/captchaverify.js index eec25ccd..21ee4de9 100644 --- a/helpers/captchaverify.js +++ b/helpers/captchaverify.js @@ -7,7 +7,8 @@ const Captchas = require(__dirname+'/../db/captchas.js') module.exports = async (req, res, next) => { - if (!res.locals.board.settings.captcha) { + //skip captcha if disabled on board for posts only + if (res.locals.board && req.path === `/board/${res.locals.board._id}/post` && !res.locals.board.settings.captcha) { return next(); } From 59f57da1455c7b67e6531964de52621902969c17 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 11:59:29 +0000 Subject: [PATCH 25/27] limit number of reports on a post --- models/forms/report-post.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/forms/report-post.js b/models/forms/report-post.js index c5a0f4de..32135fc4 100644 --- a/models/forms/report-post.js +++ b/models/forms/report-post.js @@ -15,7 +15,10 @@ module.exports = (req, posts) => { message: `Reported ${posts.length} post(s)`, action: '$push', query: { - 'reports': report + 'reports': { + '$each': [report], + '$slice': -5 //limit number of reports + } } }; From a56b9b9fadf2c72d790f05fc57f6664f64cf0ef4 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 12:00:20 +0000 Subject: [PATCH 26/27] fix building first page whenall posts/only first page posts deleted, and redirect user immediately after new post -- no need to wait for many pages to be rebuilt, improves perceived posting speed --- build.js | 10 ++++++++-- models/forms/make-post.js | 20 +++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/build.js b/build.js index 12e6ab27..303dc3c5 100644 --- a/build.js +++ b/build.js @@ -16,7 +16,7 @@ module.exports = { }, buildThread: async (threadId, board) => { -//console.log('building thread', `${board._id}/thread/${threadId}.html`); +//console.log('building thread', `${board._id || board}/thread/${threadId}.html`); if (!board._id) { board = await Boards.findOne(board); } @@ -47,7 +47,13 @@ module.exports = { //building multiple pages (for rebuilds) buildBoardMultiple: async (board, startpage=1, endpage=10) => { const maxPage = Math.ceil((await Posts.getPages(board._id)) / 10); - endpage = maxPage < endpage ? maxPage : endpage; + if (endpage === 0) { + //deleted only/all posts, so only 1 page will remain + endpage = 1; + } else if (maxPage < endpage) { + //else just build up to the max page if it is greater than input page number + endpage = maxPage + } const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0 const threads = await Posts.getRecent(board._id, startpage, difference*10); const buildArray = []; diff --git a/models/forms/make-post.js b/models/forms/make-post.js index aa0a1f58..e578d6e4 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -234,14 +234,15 @@ module.exports = async (req, res, next, numFiles) => { } const postId = await Posts.insertOne(req.params.board, data, thread); + const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`; + + //build just the thread they need to see first and send them immediately + await buildThread(data.thread || postId, res.locals.board); + res.redirect(successRedirect); - //now we need to rebuild pages + //now rebuild other pages const parallelPromises = [] - //always need to rebuild catalog - parallelPromises.push(buildCatalog(res.locals.board)); if (data.thread) { - //new reply, so build the thread first - parallelPromises.push(buildThread(thread.postId, res.locals.board)); //refersh pages const threadPage = await Posts.getThreadPage(req.params.board, thread); if (data.email === 'sage') { @@ -252,17 +253,18 @@ module.exports = async (req, res, next, numFiles) => { parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPage)); } } else { - //new thread, rebuild all pages and prune old threads + //new thread, rebuild all pages and prunes old threads const prunedThreads = await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit); for (let i = 0; i < prunedThreads.length; i++) { parallelPromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`)); } parallelPromises.push(buildBoardMultiple(res.locals.board, 1, 10)); } - await Promise.all(parallelPromises); - const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`; + //always rebuild catalog for post counts and ordering + parallelPromises.push(buildCatalog(res.locals.board)); - return res.redirect(successRedirect); + //finish building other pages + await Promise.all(parallelPromises); } From 476cf5c2a97553e4256b8fbfa45f88aa8ad781c6 Mon Sep 17 00:00:00 2001 From: fatchan Date: Tue, 21 May 2019 12:12:34 +0000 Subject: [PATCH 27/27] always thumbnail animated images to prevent large filesize by small dimension gifs not being thumbnailed --- helpers/files/file-check-mime-types.js | 5 ++++- models/forms/make-post.js | 6 ++++-- models/forms/uploadbanners.js | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/helpers/files/file-check-mime-types.js b/helpers/files/file-check-mime-types.js index c6b438f9..2f1eedad 100644 --- a/helpers/files/file-check-mime-types.js +++ b/helpers/files/file-check-mime-types.js @@ -5,6 +5,9 @@ const imageMimeTypes = new Set([ 'image/pjpeg', 'image/png', 'image/bmp', +]); + +const animatedImageMimeTypes = new Set([ 'image/gif', 'image/webp', ]); @@ -16,6 +19,6 @@ const videoMimeTypes = new Set([ module.exports = (mimetype, options) => { - return (options.video && videoMimeTypes.has(mimetype)) || (options.image && imageMimeTypes.has(mimetype)); + return (options.video && videoMimeTypes.has(mimetype)) || (options.image && imageMimeTypes.has(mimetype) || options.animatedImage && animatedImageMimeTypes.has(mimetype)); }; diff --git a/models/forms/make-post.js b/models/forms/make-post.js index e578d6e4..ca8e9d03 100644 --- a/models/forms/make-post.js +++ b/models/forms/make-post.js @@ -77,7 +77,7 @@ module.exports = async (req, res, next, numFiles) => { if (numFiles > 0) { // check all mime types befoer we try saving anything for (let i = 0; i < numFiles; i++) { - if (!fileCheckMimeType(req.files.file[i].mimetype, {image: true, video: true})) { + if (!fileCheckMimeType(req.files.file[i].mimetype, {animatedImage: true, image: true, video: true})) { return res.status(400).render('message', { 'title': 'Bad request', 'message': `Invalid file type for ${req.files.file[i].name}. Mimetype ${req.files.file[i].mimetype} not allowed.`, @@ -109,7 +109,9 @@ module.exports = async (req, res, next, numFiles) => { processedFile.geometry = imageData.size // object with width and height pixels processedFile.sizeString = formatSize(processedFile.size) // 123 Ki string processedFile.geometryString = imageData.Geometry // 123 x 123 string - if (processedFile.geometry.height <= 128 && processedFile.geometry.width <= 128) { + if (fileCheckMimeType(file.mimetype, {image: true}) //always thumbnail gif/webp + && processedFile.geometry.height <= 128 + && processedFile.geometry.width <= 128) { processedFile.hasThumb = false; } else { processedFile.hasThumb = true; diff --git a/models/forms/uploadbanners.js b/models/forms/uploadbanners.js index bbf26f80..980b31ba 100644 --- a/models/forms/uploadbanners.js +++ b/models/forms/uploadbanners.js @@ -16,7 +16,7 @@ module.exports = async (req, res, next, numFiles) => { // check all mime types befoer we try saving anything for (let i = 0; i < numFiles; i++) { - if (!fileCheckMimeType(req.files.file[i].mimetype, {image: true, video: false})) { + if (!fileCheckMimeType(req.files.file[i].mimetype, {image: true, animatedImage: true, video: false})) { return res.status(400).render('message', { 'title': 'Bad request', 'message': `Invalid file type for ${req.files.file[i].name}. Mimetype ${req.files.file[i].mimetype} not allowed.`,