From 7868e1c5d364692ede046ab99d374a9a4d06835f Mon Sep 17 00:00:00 2001 From: fatchan Date: Thu, 29 Aug 2019 08:50:29 +0000 Subject: [PATCH] use a queue with workers for generating static pages in background that arent immediately needed --- README.md | 5 +- configs/main.json.example | 1 + ecosystem.config.js | 16 +++- helpers/build.js | 128 +++++++++++++--------------- models/forms/actionhandler.js | 88 ++++++++++++++++--- models/forms/addnews.js | 7 +- models/forms/changeboardsettings.js | 22 +++-- models/forms/deletebanners.js | 11 ++- models/forms/deletenews.js | 7 +- models/forms/makepost.js | 66 +++++++++----- models/forms/uploadbanners.js | 13 ++- models/pages/banners.js | 2 +- models/pages/board.js | 6 +- models/pages/catalog.js | 2 +- models/pages/modlog.js | 7 +- models/pages/modloglist.js | 2 +- models/pages/thread.js | 5 +- package-lock.json | 43 ++++++++++ package.json | 1 + queue.js | 32 +++++++ schedules.js | 19 ++--- worker.js | 56 ++++++++++++ 22 files changed, 406 insertions(+), 133 deletions(-) create mode 100644 queue.js create mode 100644 worker.js diff --git a/README.md b/README.md index 8d733a69..0eaf8e52 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,10 @@ Please note: ##### Requirements - Linux (most likely could work elsewhere, but why?) - Node.js (to run the app) +- MongoDB (database, duh) +- Redis (queues, and eventually for caching and mutex/locking) - Nginx (handle https, serve static content and html) - Certbot/letsencrypt (for https cert) -- MongoDB (database, duh) - Imagemagick (thumbnailing images) - Ffmpeg (thumbnailing videos) - Bcrypt (account password hashes) @@ -55,6 +56,8 @@ $ sudo apt-get install bcrypt nginx ffmpeg imagemagick [Install](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-debian/#install-mongodb-community-edition-on-debian) and [configure auth for mongodb](https://medium.com/mongoaudit/how-to-enable-authentication-on-mongodb-b9e8a924efac). This is to avoid out of date verisons in debian repos. +[Install and configure](https://www.digitalocean.com/community/tutorials/how-to-install-and-secure-redis-on-debian-9) Redis. + Install nodejs. You can use [node version manager](https://github.com/nvm-sh/nvm) (nvm) to help with this. Once you have nvm, install the LTS version of nodejs (currently 10.x). ```bash diff --git a/configs/main.json.example b/configs/main.json.example index 285c7fdb..ab1d4790 100644 --- a/configs/main.json.example +++ b/configs/main.json.example @@ -5,6 +5,7 @@ "tripcodeSecret": "long random string", "ipHashSecret": "long random string", "postPasswordSecret": "long random string", + "redisPassword": "long random string", "cacheTemplates": true, "refererCheck": false, "refererRegex": "^https?:\\/\\/(?:www\\.)?domain\\.com\\/", diff --git a/ecosystem.config.js b/ecosystem.config.js index 12046eec..a807287e 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -15,10 +15,24 @@ module.exports = { env_production: { NODE_ENV: 'production' } + }, { + name: 'build-worker', + script: 'worker.js', + instances: 1, //could increase if building is getting backed up + autorestart: true, + watch: false, + max_memory_restart: '1G', + log_date_format: 'YYYY-MM-DD HH:mm:ss.SSS', + env: { + NODE_ENV: 'development' + }, + env_production: { + NODE_ENV: 'production' + } }, { name: 'chan', script: 'server.js', - instances: 0, //0 = number of cpu cores + instances: 0, // 0 = number of cpu cores autorestart: true, watch: false, max_memory_restart: '1G', diff --git a/helpers/build.js b/helpers/build.js index a9e13ab8..6e2750f3 100644 --- a/helpers/build.js +++ b/helpers/build.js @@ -3,98 +3,95 @@ const Mongo = require(__dirname+'/../db/db.js') , msTime = require(__dirname+'/mstime.js') , { Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/') - , render = require(__dirname+'/render.js'); + , render = require(__dirname+'/render.js') + , timeDiffString = (label, end) => `${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`; module.exports = { - buildBanners: async(board) => { - const label = `/${board._id}/banners.html`; + buildBanners: async (options) => { + const label = `/${options.board._id}/banners.html`; const start = process.hrtime(); - const html = render(label, 'banners.pug', { - board: board, - }); + const html = render(label, 'banners.pug', options); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, - buildCatalog: async (board) => { - const label = `/${board._id || board}/catalog.html`; + buildCatalog: async (options) => { + const label = `/${options.board._id || options.board}/catalog.html`; const start = process.hrtime(); - if (!board._id) { - board = await Boards.findOne(board); + if (!options.board._id) { + options.board = await Boards.findOne(options.board); } - const threads = await Posts.getCatalog(board._id); + const threads = await Posts.getCatalog(options.board._id); const html = render(label, 'catalog.pug', { - board, + ...options, threads, }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, - buildThread: async (threadId, board) => { - const label = `/${board._id || board}/thread/${threadId}.html`; + buildThread: async (options) => { + const label = `/${options.board._id || options.board}/thread/${options.threadId}.html`; const start = process.hrtime(); - if (!board._id) { - board = await Boards.findOne(board); + if (!options.board._id) { + options.board = await Boards.findOne(options.board); } - const thread = await Posts.getThread(board._id, threadId) + const thread = await Posts.getThread(options.board._id, options.threadId); if (!thread) { return; //this thread may have been an OP that was deleted } const html = render(label, 'thread.pug', { - board, + ...options, thread, }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, - buildBoard: async (board, page, maxPage=null) => { - const label = `/${board._id}/${page === 1 ? 'index' : page}.html`; + buildBoard: async (options) => { + const label = `/${options.board._id}/${options.page === 1 ? 'index' : options.page}.html`; const start = process.hrtime(); - const threads = await Posts.getRecent(board._id, page); - if (maxPage == null) { - maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10)); + const threads = await Posts.getRecent(options.board._id, options.page); + if (!options.maxPage) { + options.maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10)); } const html = render(label, 'board.pug', { - board, + ...options, threads, - maxPage, - page, }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, //building multiple pages (for rebuilds) - buildBoardMultiple: async (board, startpage=1, endpage) => { + buildBoardMultiple: async (options) => { const start = process.hrtime(); - const maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10)); - if (endpage === 0) { + const maxPage = Math.min(Math.ceil((await Posts.getPages(options.board._id)) / 10), Math.ceil(options.board.settings.threadLimit/10)); + if (options.endpage === 0) { //deleted only/all posts, so only 1 page will remain - endpage = 1; - } else if (maxPage < endpage) { + options.endpage = 1; + } else if (maxPage < options.endpage) { //else just build up to the max page if it is greater than input page number - endpage = maxPage + options.endpage = maxPage } - const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0 - const threads = await Posts.getRecent(board._id, startpage, difference*10); - const label = `/${board._id}/${startpage === 1 ? 'index' : startpage}.html => /${board._id}/${endpage === 1 ? 'index' : endpage}.html`; + const difference = options.endpage-options.startpage + 1; //+1 because for single pagemust be > 0 + const threads = await Posts.getRecent(options.board._id, options.startpage, difference*10); + const label = `/${options.board._id}/${options.startpage === 1 ? 'index' : options.startpage}.html => /${options.board._id}/${options.endpage === 1 ? 'index' : options.endpage}.html`; const buildArray = []; - for (let i = startpage; i <= endpage; i++) { + for (let i = options.startpage; i <= options.endpage; i++) { let spliceStart = (i-1)*10; if (spliceStart > 0) { spliceStart = spliceStart - 1; } buildArray.push( - render(`${board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', { - board, + render(`${options.board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', { + board: options.board, threads: threads.splice(0,10), maxPage, page: i, @@ -103,7 +100,7 @@ module.exports = { } await Promise.all(buildArray); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); }, buildNews: async () => { @@ -114,46 +111,43 @@ module.exports = { news }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, - buildModLog: async (board, startDate, endDate, logs) => { - if (!startDate || !endDate) { - startDate = new Date(); //this is being built by action handler so will always be current date - endDate = new Date(startDate.getTime()); - startDate.setHours(0,0,0,0); - endDate.setHours(23,59,59,999); + buildModLog: async (options) => { + if (!options.startDate || !options.endDate) { + options.startDate = new Date(); //this is being built by action handler so will always be current date + options.endDate = new Date(options.startDate.getTime()); + options.startDate.setHours(0,0,0,0); + options.endDate.setHours(23,59,59,999); } - const day = ('0'+startDate.getDate()).slice(-2); - const month = ('0'+(startDate.getMonth()+1)).slice(-2); - const year = startDate.getFullYear(); - const label = `/${board._id}/logs/${month}-${day}-${year}.html`; + const day = ('0'+options.startDate.getDate()).slice(-2); + const month = ('0'+(options.startDate.getMonth()+1)).slice(-2); + const year = options.startDate.getFullYear(); + const label = `/${options.board._id}/logs/${month}-${day}-${year}.html`; const start = process.hrtime(); - if (!logs) { - logs = await Modlogs.findBetweenDate(board, startDate, endDate); + if (!options.logs) { + options.logs = await Modlogs.findBetweenDate(options.board, options.startDate, options.endDate); } const html = render(label, 'modlog.pug', { - board, - logs, - startDate, - endDate + ...options }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, - buildModLogList: async (board) => { - const label = `/${board._id}/logs.html`; + buildModLogList: async (options) => { + const label = `/${options.board._id}/logs.html`; const start = process.hrtime(); - const dates = await Modlogs.getDates(board); + const dates = await Modlogs.getDates(options.board); const html = render(label, 'modloglist.pug', { - board, + board: options.board, dates }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, @@ -222,7 +216,7 @@ module.exports = { fileStats, }); const end = process.hrtime(start); - console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); + console.log(timeDiffString(label, end)); return html; }, diff --git a/models/forms/actionhandler.js b/models/forms/actionhandler.js index 56a2eb74..054e9b98 100644 --- a/models/forms/actionhandler.js +++ b/models/forms/actionhandler.js @@ -14,7 +14,7 @@ const { Posts, Boards, Modlogs } = require(__dirname+'/../../db/') , dismissReports = require(__dirname+'/dismissreport.js') , { remove } = require('fs-extra') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') - , { buildModLog, buildModLogList, buildCatalog, buildThread, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js') + , buildQueue = require(__dirname+'/../../queue.js') , { postPasswordSecret } = require(__dirname+'/../../configs/main.json') , { createHash, timingSafeEqual } = require('crypto'); @@ -245,8 +245,18 @@ module.exports = async (req, res, next) => { await Modlogs.insertMany(modlogDocuments); for (let i = 0; i < threadBoards.length; i++) { const board = buildBoards[threadBoards[i]]; - parallelPromises.push(buildModLog(board)); - parallelPromises.push(buildModLogList(board)); + buildQueue.push({ + 'task': 'buildModLog', + 'options': { + 'board': res.locals.board, + } + }); + buildQueue.push({ + 'task': 'buildModLogList', + 'options': { + 'board': res.locals.board, + } + }); } } } @@ -315,7 +325,13 @@ module.exports = async (req, res, next) => { const board = buildBoards[boardName]; //rebuild impacted threads for (let j = 0; j < boardThreadMap[boardName].threads.length; j++) { - parallelPromises.push(buildThread(boardThreadMap[boardName].threads[j], board)); + buildQueue.push({ + 'task': 'buildThread', + 'options': { + 'threadId': boardThreadMap[boardName].threads[j], + 'board': board, + } + }); } //refersh any pages affected const afterPages = Math.ceil((await Posts.getPages(boardName)) / 10); @@ -329,26 +345,68 @@ module.exports = async (req, res, next) => { parallelPromises.push(remove(`${uploadDirectory}html/${boardName}/${k}.html`)); } } - parallelPromises.push(buildBoardMultiple(board, 1, afterPages)); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': board, + 'startpage': 1, + 'endpage': afterPages, + } + }); } else { //number of pages did not change, only possibly building existing pages const threadPageOldest = await Posts.getThreadPage(boardName, bounds.oldest); const threadPageNewest = bounds.oldest.postId === bounds.newest.postId ? threadPageOldest : await Posts.getThreadPage(boardName, bounds.newest); if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) { if (!boardThreadMap[boardName].directThreads) { - //onyl deleting posts from threads, so thread order wont change, thus we dont delete all pages after - parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest)); + //only deleting posts from threads, so thread order wont change, thus we dont delete all pages after + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': board, + 'startpage': threadPageNewest, + 'endpage': threadPageOldest, + } + }); } else { //deleting threads, so we delete all pages after - parallelPromises.push(buildBoardMultiple(board, threadPageNewest, afterPages)); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': board, + 'startpage': threadPageNewest, + 'endpage': afterPages, + } + }); } } else if (req.body.sticky) { //else if -- if deleting, other actions are not executed/irrelevant //rebuild current and newer pages - parallelPromises.push(buildBoardMultiple(board, 1, threadPageOldest)); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': board, + 'startpage': 1, + 'endpage': threadPageOldest, + } + }); } else if (req.body.lock || req.body.sage || req.body.cyclic || req.body.unlink_file) { - parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest)); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': board, + 'startpage': threadPageNewest, + 'endpage': threadPageOldest, + } + }); } else if (req.body.spoiler || req.body.ban || req.body.global_ban) { - parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest)); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': board, + 'startpage': threadPageNewest, + 'endpage': afterPages, + } + }); if (!boardThreadMap[boardName].directThreads) { catalogRebuild = false; //these actions dont affect the catalog tile since not on an OP and dont change reply/image counts @@ -357,12 +415,18 @@ module.exports = async (req, res, next) => { } if (catalogRebuild) { //the actions will affect the catalog, so we better rebuild it - parallelPromises.push(buildCatalog(board)); + buildQueue.push({ + 'task': 'buildCatalog', + 'options': { + 'board': board, + } + }); } } } if (parallelPromises.length > 0) { + //since queue changes, this just removing old html files await Promise.all(parallelPromises); } diff --git a/models/forms/addnews.js b/models/forms/addnews.js index 0ec3a677..62c6a21e 100644 --- a/models/forms/addnews.js +++ b/models/forms/addnews.js @@ -2,7 +2,7 @@ const { News } = require(__dirname+'/../../db/') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') - , { buildNews } = require(__dirname+'/../../helpers/build.js') + , buildQueue = require(__dirname+'/../../queue.js') , linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js') , simpleMarkdown = require(__dirname+'/../../helpers/posting/markdown.js') , escape = require(__dirname+'/../../helpers/posting/escape.js') @@ -27,7 +27,10 @@ module.exports = async (req, res, next) => { await News.insertOne(post); - await buildNews(); + buildQueue.push({ + 'task': 'buildNews', + 'options': {} + }); return res.render('message', { 'title': 'Success', diff --git a/models/forms/changeboardsettings.js b/models/forms/changeboardsettings.js index 1337e308..9316c953 100644 --- a/models/forms/changeboardsettings.js +++ b/models/forms/changeboardsettings.js @@ -2,7 +2,7 @@ const { Boards, Posts, Accounts } = require(__dirname+'/../../db/') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') - , { buildHomepage, buildCatalog, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js') + , buildQueue = require(__dirname+'/../../queue.js') , { remove } = require('fs-extra') , deletePosts = require(__dirname+'/deletepost.js') , linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js') @@ -101,15 +101,27 @@ module.exports = async (req, res, next) => { for (let i = newMaxPage+1; i <= oldMaxPage; i++) { promises.push(remove(`${uploadDirectory}html/${req.params.board}/${i}.html`)); } - //rebuild valid board pages for page numbers, and catalog for prunedthreads - promises.push(buildBoardMultiple(res.locals.board, 1, newMaxPage)); - promises.push(buildCatalog(res.locals.board)); + //rebuild all board pages for page nav numbers, and catalog + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': res.locals.board, + 'startpage': 1, + 'endpage': newMaxPage + } + }); + buildQueue.push({ + 'task': 'buildCatalog', + 'options': { + 'board': res.locals.board, + } + }); } } if (newSettings.captchaMode !== oldSettings.captchaMode) { - //TODO: only remove necessary pages here promises.push(remove(`${uploadDirectory}html/${req.params.board}/`)); +//TODO: dont remove all pages, only remove some and add important pages to build queue here } if (promises.length > 0) { diff --git a/models/forms/deletebanners.js b/models/forms/deletebanners.js index 43705ced..3612d62c 100644 --- a/models/forms/deletebanners.js +++ b/models/forms/deletebanners.js @@ -3,7 +3,7 @@ const { remove } = require('fs-extra') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') , { Boards } = require(__dirname+'/../../db/') - , { buildBanners } = require(__dirname+'/../../helpers/build.js') + , buildQueue = require(__dirname+'/../../queue.js'); module.exports = async (req, res, next) => { @@ -15,7 +15,7 @@ module.exports = async (req, res, next) => { })); //remove from db - await Boards.removeBanners(req.params.board, req.body.checkedbanners); + const amount = await Boards.removeBanners(req.params.board, req.body.checkedbanners); //update res locals banners in memory res.locals.board.banners = res.locals.board.banners.filter(banner => { @@ -23,7 +23,12 @@ module.exports = async (req, res, next) => { }); //rebuild public banners page - await buildBanners(res.locals.board); + buildQueue.push({ + 'task': 'buildBanners', + 'options': { + 'board': res.locals.board, + } + }); return res.render('message', { 'title': 'Success', diff --git a/models/forms/deletenews.js b/models/forms/deletenews.js index 5ee374c8..ed9b879e 100644 --- a/models/forms/deletenews.js +++ b/models/forms/deletenews.js @@ -1,13 +1,16 @@ 'use strict'; const { News } = require(__dirname+'/../../db/') - , { buildNews } = require(__dirname+'/../../helpers/build.js') + , buildQueue = require(__dirname+'/../../queue.js') module.exports = async (req, res, next) => { await News.deleteMany(req.body.checkednews); - await buildNews(); + buildQueue.push({ + 'task': 'buildNews', + 'options': {} + }); return res.render('message', { 'title': 'Success', diff --git a/models/forms/makepost.js b/models/forms/makepost.js index cd2e5a66..98aaccbe 100644 --- a/models/forms/makepost.js +++ b/models/forms/makepost.js @@ -27,20 +27,21 @@ const path = require('path') , deletePosts = require(__dirname+'/deletepost.js') , spamCheck = require(__dirname+'/../../helpers/checks/spamcheck.js') , { postPasswordSecret } = require(__dirname+'/../../configs/main.json') - , { buildCatalog, buildThread, buildBoard, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js'); + , buildQueue = require(__dirname+'/../../queue.js') + , { buildThread } = require(__dirname+'/../../helpers/build.js'); module.exports = async (req, res, next) => { //spam/flood check - const flood = await spamCheck(req, res); - if (flood) { - deleteTempFiles(req).catch(e => console.error); - return res.status(429).render('message', { - 'title': 'Flood detected', - 'message': 'Please wait before making another post, or a post similar to another user', - 'redirect': `/${req.params.board}${req.body.thread ? '/thread/' + req.body.thread + '.html' : ''}` - }); - } + const flood = await spamCheck(req, res); + if (flood) { + deleteTempFiles(req).catch(e => console.error); + return res.status(429).render('message', { + 'title': 'Flood detected', + 'message': 'Please wait before making another post, or a post similar to another user', + 'redirect': `/${req.params.board}${req.body.thread ? '/thread/' + req.body.thread + '.html' : ''}` + }); + } // check if this is responding to an existing thread let redirect = `/${req.params.board}/` @@ -395,23 +396,37 @@ module.exports = async (req, res, next) => { } const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`; - console.log(`NEW POST -> ${successRedirect}`); //build just the thread they need to see first and send them immediately - await buildThread(data.thread || postId, res.locals.board); + await buildThread({ + 'threadId': data.thread || postId, + 'board': res.locals.board + }); res.redirect(successRedirect); - //now rebuild other pages - const parallelPromises = []; + //now add other pages to be built in background if (data.thread) { //refersh pages const threadPage = await Posts.getThreadPage(req.params.board, thread); if (data.email === 'sage' || thread.sage) { //refresh the page that the thread is on - parallelPromises.push(buildBoard(res.locals.board, threadPage)); + buildQueue.push({ + 'task': 'buildBoard', + 'options': { + 'board': res.locals.board, + 'page': threadPage + } + }); } else { //if not saged, it will bump so we should refresh any pages above it as well - parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPage)); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': res.locals.board, + 'startpage': 1, + 'endpage': threadPage + } + }); } } else { //new thread, prunes any old threads before rebuilds @@ -419,13 +434,22 @@ module.exports = async (req, res, next) => { if (prunedThreads.length > 0) { await deletePosts(prunedThreads, req.params.board); } - parallelPromises.push(buildBoardMultiple(res.locals.board, 1, Math.ceil(threadLimit/10))); + buildQueue.push({ + 'task': 'buildBoardMultiple', + 'options': { + 'board': res.locals.board, + 'startpage': 1, + 'endpage': Math.ceil(threadLimit/10) + } + }); } //always rebuild catalog for post counts and ordering - parallelPromises.push(buildCatalog(res.locals.board)); - - //finish building other pages - await Promise.all(parallelPromises); + buildQueue.push({ + 'task': 'buildCatalog', + 'options': { + 'board': res.locals.board, + } + }); } diff --git a/models/forms/uploadbanners.js b/models/forms/uploadbanners.js index c78ca317..85b34c00 100644 --- a/models/forms/uploadbanners.js +++ b/models/forms/uploadbanners.js @@ -8,7 +8,7 @@ const path = require('path') , imageIdentify = require(__dirname+'/../../helpers/files/imageidentify.js') , deleteTempFiles = require(__dirname+'/../../helpers/files/deletetempfiles.js') , { Boards } = require(__dirname+'/../../db/') - , { buildBanners } = require(__dirname+'/../../helpers/build.js') + , buildQueue = require(__dirname+'/../../queue.js'); module.exports = async (req, res, next) => { @@ -79,8 +79,15 @@ module.exports = async (req, res, next) => { //add banners to board in memory res.locals.board.banners = res.locals.board.banners.concat(filenames); - // rebuild the public banners page - await buildBanners(res.locals.board); + if (filenames.length > 0) { + //add public banners page to build queue + buildQueue.push({ + 'task': 'buildBanners', + 'options': { + 'board': res.locals.board, + } + }); + } return res.render('message', { 'title': 'Success', diff --git a/models/pages/banners.js b/models/pages/banners.js index 75d8da6d..eadb912f 100644 --- a/models/pages/banners.js +++ b/models/pages/banners.js @@ -6,7 +6,7 @@ module.exports = async (req, res, next) => { let html; try { - html = await buildBanners(res.locals.board); + html = await buildBanners({ board: res.locals.board }); } catch (err) { return next(err); } diff --git a/models/pages/board.js b/models/pages/board.js index c39456b7..13d13240 100644 --- a/models/pages/board.js +++ b/models/pages/board.js @@ -12,7 +12,11 @@ module.exports = async (req, res, next) => { if (page > maxPage) { return next(); } - html = await buildBoard(res.locals.board, page, maxPage); + html = await buildBoard({ + board: res.locals.board, + page, + maxPage + }); } catch (err) { return next(err); } diff --git a/models/pages/catalog.js b/models/pages/catalog.js index 003bd03f..51947120 100644 --- a/models/pages/catalog.js +++ b/models/pages/catalog.js @@ -6,7 +6,7 @@ module.exports = async (req, res, next) => { let html; try { - html = await buildCatalog(res.locals.board); + html = await buildCatalog({ board: res.locals.board }); } catch (err) { return next(err); } diff --git a/models/pages/modlog.js b/models/pages/modlog.js index 6774f394..0694ba15 100644 --- a/models/pages/modlog.js +++ b/models/pages/modlog.js @@ -19,7 +19,12 @@ module.exports = async (req, res, next) => { if (!logs || logs.length === 0) { return next(); } - html = await buildModLog(res.locals.board, startDate, endDate, logs); + html = await buildModLog({ + board: res.locals.board, + startDate, + endDate, + logs + }); } catch (err) { return next(err); } diff --git a/models/pages/modloglist.js b/models/pages/modloglist.js index 8badd924..0b32d3ca 100644 --- a/models/pages/modloglist.js +++ b/models/pages/modloglist.js @@ -6,7 +6,7 @@ module.exports = async (req, res, next) => { let html; try { - html = await buildModLogList(res.locals.board); + html = await buildModLogList({ board: res.locals.board }); } catch (err) { return next(err); } diff --git a/models/pages/thread.js b/models/pages/thread.js index ad354c22..60724e23 100644 --- a/models/pages/thread.js +++ b/models/pages/thread.js @@ -6,7 +6,10 @@ module.exports = async (req, res, next) => { let html; try { - html = await buildThread(res.locals.thread.postId, res.locals.board); + html = await buildThread({ + thredId: res.locals.thread.postId, + board: res.locals.board + }); } catch (err) { return next(err); } diff --git a/package-lock.json b/package-lock.json index e6deeb9e..d19414fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -262,6 +262,14 @@ "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.4.tgz", "integrity": "sha1-h3L80EGOPNLMFxVV1zAHQVBR9LI=" }, + "@types/redis": { + "version": "2.8.13", + "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.13.tgz", + "integrity": "sha512-p86cm5P6DMotUqCS6odQRz0JJwc5QXZw9eyH0ALVIqmq12yqtex5ighWyGFHKxak9vaA/GF/Ilu0KZ0MuXXUbg==", + "requires": { + "@types/node": "*" + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1778,6 +1786,11 @@ "domelementtype": "1" } }, + "double-ended-queue": { + "version": "2.1.0-0", + "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", + "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" + }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -5638,6 +5651,26 @@ "resolve": "^1.1.6" } }, + "redis": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", + "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", + "requires": { + "double-ended-queue": "^2.1.0-0", + "redis-commands": "^1.2.0", + "redis-parser": "^2.6.0" + } + }, + "redis-commands": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" + }, + "redis-parser": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", + "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" + }, "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", @@ -5842,6 +5875,16 @@ "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz", "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w=" }, + "rsmq": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/rsmq/-/rsmq-0.11.0.tgz", + "integrity": "sha512-Uo+jQ85T3jbajAwoU7MGyIVRC1ytUDfMV+yjskQzIMEJDLWpcJmw+DlacM3EXwjwD9g4Q2yWdKuGXi8C2V8WIg==", + "requires": { + "@types/redis": "^2.8.0", + "lodash": "^4.17.11", + "redis": "^2.8.0" + } + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", diff --git a/package.json b/package.json index 047caea8..934d0fd3 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "path": "^0.12.7", "pm2": "^3.5.1", "pug": "^2.0.4", + "rsmq": "^0.11.0", "sanitize-html": "^1.20.1", "saslprep": "^1.0.3" }, diff --git a/queue.js b/queue.js new file mode 100644 index 00000000..41df9a84 --- /dev/null +++ b/queue.js @@ -0,0 +1,32 @@ +const RedisSMQ = require('rsmq') + , configs = require(__dirname+'/configs/main.json') + , rsmq = new RedisSMQ({ host: '127.0.0.1', port: 6379, ns: 'rsmq', password: configs.redisPassword }) + , queuename = 'generate' + +rsmq.createQueue({ qname: queuename }, (err) => { + if (err && err.name !== 'queueExists') { + return console.error(err); + } +}); + +module.exports.push = (data) => { + rsmq.sendMessage({ qname: queuename, message: JSON.stringify(data) }, (err) => { + if (err) { + return console.error(err); + } + //message enqueued successfully + }); +} + +/* +//was testing +setInterval(() => { + const data = { + task: 'buildCatalog', + options: { + 'board': 'b' + } + } + module.exports.push(data); +}, 500); +*/ diff --git a/schedules.js b/schedules.js index 485d5503..756216f7 100644 --- a/schedules.js +++ b/schedules.js @@ -7,24 +7,23 @@ process const msTime = require(__dirname+'/helpers/mstime.js') , deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js') , Mongo = require(__dirname+'/db/db.js') - , Mutex = require(__dirname+'/mutex.js'); + , Mutex = require(__dirname+'/mutex.js') + , buildQueue = require(__dirname+'/queue.js'); (async () => { await Mongo.connect(); await Mutex.connect(); - const { buildHomepage } = require(__dirname+'/helpers/build.js') - , Files = require(__dirname+'/db/files.js'); + const Files = require(__dirname+'/db/files.js'); console.log('Starting schedules'); - setInterval(async () => { - try { - await buildHomepage(); - } catch (e) { - console.error(e); - } - }, msTime.minute*5); //rebuild homepage for pph updates +// setInterval(async () => { + buildQueue.push({ + 'task': 'buildHomepage', + 'options': {} + }) +// }, msTime.minute*5); //rebuild homepage for pph updates setInterval(async () => { try { diff --git a/worker.js b/worker.js new file mode 100644 index 00000000..9f0eb47e --- /dev/null +++ b/worker.js @@ -0,0 +1,56 @@ +'use strict'; + +process + .on('uncaughtException', console.error) + .on('unhandledRejection', console.error); + +const RedisSMQ = require('rsmq') + , configs = require(__dirname+'/configs/main.json') + , rsmq = new RedisSMQ({ host: '127.0.0.1', port: 6379, ns: 'rsmq', password: configs.redisPassword }) + , queuename = 'generate' + , Mongo = require(__dirname+'/db/db.js') + , Mutex = require(__dirname+'/mutex.js'); + +let buildTasks = {} + , interval = 100; + +(async () => { + + await Mongo.connect(); + await Mutex.connect(); + buildTasks = require(__dirname+'/helpers/build.js'); + + rsmq.createQueue({ qname: queuename }, (err) => { + if (err && err.name !== 'queueExists') { + return console.error(err); + } + setTimeout(processQueueLoop, interval); + }); + +})(); + + +function processQueueLoop() { + rsmq.receiveMessage({ qname: queuename }, async (err, resp) => { + if (err) { + return console.error(err); + } + if (resp.id) { + interval = 100; //keeps queue checking fast when there are tasks + const message = JSON.parse(resp.message); + await buildTasks[message.task](message.options); + rsmq.deleteMessage({ qname: queuename, id: resp.id }, (err) => { + if (err) { + return console.error(err); + } + //message deleted successfully + }); + } else { + //max 2 sec poll time + if (interval < 2000) { //slow down queue when empty + interval += 100; + } + } + setTimeout(processQueueLoop, interval); + }); +}