use a queue with workers for generating static pages in background that arent immediately needed

merge-requests/208/head
fatchan 5 years ago
parent 52b37fb34b
commit 7868e1c5d3
  1. 5
      README.md
  2. 1
      configs/main.json.example
  3. 14
      ecosystem.config.js
  4. 130
      helpers/build.js
  5. 88
      models/forms/actionhandler.js
  6. 7
      models/forms/addnews.js
  7. 22
      models/forms/changeboardsettings.js
  8. 11
      models/forms/deletebanners.js
  9. 7
      models/forms/deletenews.js
  10. 48
      models/forms/makepost.js
  11. 13
      models/forms/uploadbanners.js
  12. 2
      models/pages/banners.js
  13. 6
      models/pages/board.js
  14. 2
      models/pages/catalog.js
  15. 7
      models/pages/modlog.js
  16. 2
      models/pages/modloglist.js
  17. 5
      models/pages/thread.js
  18. 43
      package-lock.json
  19. 1
      package.json
  20. 32
      queue.js
  21. 19
      schedules.js
  22. 56
      worker.js

@ -40,9 +40,10 @@ Please note:
##### Requirements ##### Requirements
- Linux (most likely could work elsewhere, but why?) - Linux (most likely could work elsewhere, but why?)
- Node.js (to run the app) - Node.js (to run the app)
- MongoDB (database, duh)
- Redis (queues, and eventually for caching and mutex/locking)
- Nginx (handle https, serve static content and html) - Nginx (handle https, serve static content and html)
- Certbot/letsencrypt (for https cert) - Certbot/letsencrypt (for https cert)
- MongoDB (database, duh)
- Imagemagick (thumbnailing images) - Imagemagick (thumbnailing images)
- Ffmpeg (thumbnailing videos) - Ffmpeg (thumbnailing videos)
- Bcrypt (account password hashes) - Bcrypt (account password hashes)
@ -55,6 +56,8 @@ $ sudo apt-get install bcrypt nginx ffmpeg imagemagick
[Install](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-debian/#install-mongodb-community-edition-on-debian) and [configure auth for mongodb](https://medium.com/mongoaudit/how-to-enable-authentication-on-mongodb-b9e8a924efac). This is to avoid out of date verisons in debian repos. [Install](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-debian/#install-mongodb-community-edition-on-debian) and [configure auth for mongodb](https://medium.com/mongoaudit/how-to-enable-authentication-on-mongodb-b9e8a924efac). This is to avoid out of date verisons in debian repos.
[Install and configure](https://www.digitalocean.com/community/tutorials/how-to-install-and-secure-redis-on-debian-9) Redis.
Install nodejs. You can use [node version manager](https://github.com/nvm-sh/nvm) (nvm) to help with this. Install nodejs. You can use [node version manager](https://github.com/nvm-sh/nvm) (nvm) to help with this.
Once you have nvm, install the LTS version of nodejs (currently 10.x). Once you have nvm, install the LTS version of nodejs (currently 10.x).
```bash ```bash

@ -5,6 +5,7 @@
"tripcodeSecret": "long random string", "tripcodeSecret": "long random string",
"ipHashSecret": "long random string", "ipHashSecret": "long random string",
"postPasswordSecret": "long random string", "postPasswordSecret": "long random string",
"redisPassword": "long random string",
"cacheTemplates": true, "cacheTemplates": true,
"refererCheck": false, "refererCheck": false,
"refererRegex": "^https?:\\/\\/(?:www\\.)?domain\\.com\\/", "refererRegex": "^https?:\\/\\/(?:www\\.)?domain\\.com\\/",

@ -15,6 +15,20 @@ module.exports = {
env_production: { env_production: {
NODE_ENV: 'production' NODE_ENV: 'production'
} }
}, {
name: 'build-worker',
script: 'worker.js',
instances: 1, //could increase if building is getting backed up
autorestart: true,
watch: false,
max_memory_restart: '1G',
log_date_format: 'YYYY-MM-DD HH:mm:ss.SSS',
env: {
NODE_ENV: 'development'
},
env_production: {
NODE_ENV: 'production'
}
}, { }, {
name: 'chan', name: 'chan',
script: 'server.js', script: 'server.js',

@ -3,98 +3,95 @@
const Mongo = require(__dirname+'/../db/db.js') const Mongo = require(__dirname+'/../db/db.js')
, msTime = require(__dirname+'/mstime.js') , msTime = require(__dirname+'/mstime.js')
, { Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/') , { Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/')
, render = require(__dirname+'/render.js'); , render = require(__dirname+'/render.js')
, timeDiffString = (label, end) => `${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`;
module.exports = { module.exports = {
buildBanners: async(board) => { buildBanners: async (options) => {
const label = `/${board._id}/banners.html`; const label = `/${options.board._id}/banners.html`;
const start = process.hrtime(); const start = process.hrtime();
const html = render(label, 'banners.pug', { const html = render(label, 'banners.pug', options);
board: board,
});
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
buildCatalog: async (board) => { buildCatalog: async (options) => {
const label = `/${board._id || board}/catalog.html`; const label = `/${options.board._id || options.board}/catalog.html`;
const start = process.hrtime(); const start = process.hrtime();
if (!board._id) { if (!options.board._id) {
board = await Boards.findOne(board); options.board = await Boards.findOne(options.board);
} }
const threads = await Posts.getCatalog(board._id); const threads = await Posts.getCatalog(options.board._id);
const html = render(label, 'catalog.pug', { const html = render(label, 'catalog.pug', {
board, ...options,
threads, threads,
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
buildThread: async (threadId, board) => { buildThread: async (options) => {
const label = `/${board._id || board}/thread/${threadId}.html`; const label = `/${options.board._id || options.board}/thread/${options.threadId}.html`;
const start = process.hrtime(); const start = process.hrtime();
if (!board._id) { if (!options.board._id) {
board = await Boards.findOne(board); options.board = await Boards.findOne(options.board);
} }
const thread = await Posts.getThread(board._id, threadId) const thread = await Posts.getThread(options.board._id, options.threadId);
if (!thread) { if (!thread) {
return; //this thread may have been an OP that was deleted return; //this thread may have been an OP that was deleted
} }
const html = render(label, 'thread.pug', { const html = render(label, 'thread.pug', {
board, ...options,
thread, thread,
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
buildBoard: async (board, page, maxPage=null) => { buildBoard: async (options) => {
const label = `/${board._id}/${page === 1 ? 'index' : page}.html`; const label = `/${options.board._id}/${options.page === 1 ? 'index' : options.page}.html`;
const start = process.hrtime(); const start = process.hrtime();
const threads = await Posts.getRecent(board._id, page); const threads = await Posts.getRecent(options.board._id, options.page);
if (maxPage == null) { if (!options.maxPage) {
maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10)); options.maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10));
} }
const html = render(label, 'board.pug', { const html = render(label, 'board.pug', {
board, ...options,
threads, threads,
maxPage,
page,
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
//building multiple pages (for rebuilds) //building multiple pages (for rebuilds)
buildBoardMultiple: async (board, startpage=1, endpage) => { buildBoardMultiple: async (options) => {
const start = process.hrtime(); const start = process.hrtime();
const maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10)); const maxPage = Math.min(Math.ceil((await Posts.getPages(options.board._id)) / 10), Math.ceil(options.board.settings.threadLimit/10));
if (endpage === 0) { if (options.endpage === 0) {
//deleted only/all posts, so only 1 page will remain //deleted only/all posts, so only 1 page will remain
endpage = 1; options.endpage = 1;
} else if (maxPage < endpage) { } else if (maxPage < options.endpage) {
//else just build up to the max page if it is greater than input page number //else just build up to the max page if it is greater than input page number
endpage = maxPage options.endpage = maxPage
} }
const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0 const difference = options.endpage-options.startpage + 1; //+1 because for single pagemust be > 0
const threads = await Posts.getRecent(board._id, startpage, difference*10); const threads = await Posts.getRecent(options.board._id, options.startpage, difference*10);
const label = `/${board._id}/${startpage === 1 ? 'index' : startpage}.html => /${board._id}/${endpage === 1 ? 'index' : endpage}.html`; const label = `/${options.board._id}/${options.startpage === 1 ? 'index' : options.startpage}.html => /${options.board._id}/${options.endpage === 1 ? 'index' : options.endpage}.html`;
const buildArray = []; const buildArray = [];
for (let i = startpage; i <= endpage; i++) { for (let i = options.startpage; i <= options.endpage; i++) {
let spliceStart = (i-1)*10; let spliceStart = (i-1)*10;
if (spliceStart > 0) { if (spliceStart > 0) {
spliceStart = spliceStart - 1; spliceStart = spliceStart - 1;
} }
buildArray.push( buildArray.push(
render(`${board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', { render(`${options.board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', {
board, board: options.board,
threads: threads.splice(0,10), threads: threads.splice(0,10),
maxPage, maxPage,
page: i, page: i,
@ -103,7 +100,7 @@ module.exports = {
} }
await Promise.all(buildArray); await Promise.all(buildArray);
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
}, },
buildNews: async () => { buildNews: async () => {
@ -114,46 +111,43 @@ module.exports = {
news news
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
buildModLog: async (board, startDate, endDate, logs) => { buildModLog: async (options) => {
if (!startDate || !endDate) { if (!options.startDate || !options.endDate) {
startDate = new Date(); //this is being built by action handler so will always be current date options.startDate = new Date(); //this is being built by action handler so will always be current date
endDate = new Date(startDate.getTime()); options.endDate = new Date(options.startDate.getTime());
startDate.setHours(0,0,0,0); options.startDate.setHours(0,0,0,0);
endDate.setHours(23,59,59,999); options.endDate.setHours(23,59,59,999);
} }
const day = ('0'+startDate.getDate()).slice(-2); const day = ('0'+options.startDate.getDate()).slice(-2);
const month = ('0'+(startDate.getMonth()+1)).slice(-2); const month = ('0'+(options.startDate.getMonth()+1)).slice(-2);
const year = startDate.getFullYear(); const year = options.startDate.getFullYear();
const label = `/${board._id}/logs/${month}-${day}-${year}.html`; const label = `/${options.board._id}/logs/${month}-${day}-${year}.html`;
const start = process.hrtime(); const start = process.hrtime();
if (!logs) { if (!options.logs) {
logs = await Modlogs.findBetweenDate(board, startDate, endDate); options.logs = await Modlogs.findBetweenDate(options.board, options.startDate, options.endDate);
} }
const html = render(label, 'modlog.pug', { const html = render(label, 'modlog.pug', {
board, ...options
logs,
startDate,
endDate
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
buildModLogList: async (board) => { buildModLogList: async (options) => {
const label = `/${board._id}/logs.html`; const label = `/${options.board._id}/logs.html`;
const start = process.hrtime(); const start = process.hrtime();
const dates = await Modlogs.getDates(board); const dates = await Modlogs.getDates(options.board);
const html = render(label, 'modloglist.pug', { const html = render(label, 'modloglist.pug', {
board, board: options.board,
dates dates
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },
@ -222,7 +216,7 @@ module.exports = {
fileStats, fileStats,
}); });
const end = process.hrtime(start); const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`); console.log(timeDiffString(label, end));
return html; return html;
}, },

@ -14,7 +14,7 @@ const { Posts, Boards, Modlogs } = require(__dirname+'/../../db/')
, dismissReports = require(__dirname+'/dismissreport.js') , dismissReports = require(__dirname+'/dismissreport.js')
, { remove } = require('fs-extra') , { remove } = require('fs-extra')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { buildModLog, buildModLogList, buildCatalog, buildThread, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js') , buildQueue = require(__dirname+'/../../queue.js')
, { postPasswordSecret } = require(__dirname+'/../../configs/main.json') , { postPasswordSecret } = require(__dirname+'/../../configs/main.json')
, { createHash, timingSafeEqual } = require('crypto'); , { createHash, timingSafeEqual } = require('crypto');
@ -245,8 +245,18 @@ module.exports = async (req, res, next) => {
await Modlogs.insertMany(modlogDocuments); await Modlogs.insertMany(modlogDocuments);
for (let i = 0; i < threadBoards.length; i++) { for (let i = 0; i < threadBoards.length; i++) {
const board = buildBoards[threadBoards[i]]; const board = buildBoards[threadBoards[i]];
parallelPromises.push(buildModLog(board)); buildQueue.push({
parallelPromises.push(buildModLogList(board)); 'task': 'buildModLog',
'options': {
'board': res.locals.board,
}
});
buildQueue.push({
'task': 'buildModLogList',
'options': {
'board': res.locals.board,
}
});
} }
} }
} }
@ -315,7 +325,13 @@ module.exports = async (req, res, next) => {
const board = buildBoards[boardName]; const board = buildBoards[boardName];
//rebuild impacted threads //rebuild impacted threads
for (let j = 0; j < boardThreadMap[boardName].threads.length; j++) { for (let j = 0; j < boardThreadMap[boardName].threads.length; j++) {
parallelPromises.push(buildThread(boardThreadMap[boardName].threads[j], board)); buildQueue.push({
'task': 'buildThread',
'options': {
'threadId': boardThreadMap[boardName].threads[j],
'board': board,
}
});
} }
//refersh any pages affected //refersh any pages affected
const afterPages = Math.ceil((await Posts.getPages(boardName)) / 10); const afterPages = Math.ceil((await Posts.getPages(boardName)) / 10);
@ -329,26 +345,68 @@ module.exports = async (req, res, next) => {
parallelPromises.push(remove(`${uploadDirectory}html/${boardName}/${k}.html`)); parallelPromises.push(remove(`${uploadDirectory}html/${boardName}/${k}.html`));
} }
} }
parallelPromises.push(buildBoardMultiple(board, 1, afterPages)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': 1,
'endpage': afterPages,
}
});
} else { } else {
//number of pages did not change, only possibly building existing pages //number of pages did not change, only possibly building existing pages
const threadPageOldest = await Posts.getThreadPage(boardName, bounds.oldest); const threadPageOldest = await Posts.getThreadPage(boardName, bounds.oldest);
const threadPageNewest = bounds.oldest.postId === bounds.newest.postId ? threadPageOldest : await Posts.getThreadPage(boardName, bounds.newest); const threadPageNewest = bounds.oldest.postId === bounds.newest.postId ? threadPageOldest : await Posts.getThreadPage(boardName, bounds.newest);
if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) { if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) {
if (!boardThreadMap[boardName].directThreads) { if (!boardThreadMap[boardName].directThreads) {
//onyl deleting posts from threads, so thread order wont change, thus we dont delete all pages after //only deleting posts from threads, so thread order wont change, thus we dont delete all pages after
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': threadPageOldest,
}
});
} else { } else {
//deleting threads, so we delete all pages after //deleting threads, so we delete all pages after
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, afterPages)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': afterPages,
}
});
} }
} else if (req.body.sticky) { //else if -- if deleting, other actions are not executed/irrelevant } else if (req.body.sticky) { //else if -- if deleting, other actions are not executed/irrelevant
//rebuild current and newer pages //rebuild current and newer pages
parallelPromises.push(buildBoardMultiple(board, 1, threadPageOldest)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': 1,
'endpage': threadPageOldest,
}
});
} else if (req.body.lock || req.body.sage || req.body.cyclic || req.body.unlink_file) { } else if (req.body.lock || req.body.sage || req.body.cyclic || req.body.unlink_file) {
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': threadPageOldest,
}
});
} else if (req.body.spoiler || req.body.ban || req.body.global_ban) { } else if (req.body.spoiler || req.body.ban || req.body.global_ban) {
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': afterPages,
}
});
if (!boardThreadMap[boardName].directThreads) { if (!boardThreadMap[boardName].directThreads) {
catalogRebuild = false; catalogRebuild = false;
//these actions dont affect the catalog tile since not on an OP and dont change reply/image counts //these actions dont affect the catalog tile since not on an OP and dont change reply/image counts
@ -357,12 +415,18 @@ module.exports = async (req, res, next) => {
} }
if (catalogRebuild) { if (catalogRebuild) {
//the actions will affect the catalog, so we better rebuild it //the actions will affect the catalog, so we better rebuild it
parallelPromises.push(buildCatalog(board)); buildQueue.push({
'task': 'buildCatalog',
'options': {
'board': board,
}
});
} }
} }
} }
if (parallelPromises.length > 0) { if (parallelPromises.length > 0) {
//since queue changes, this just removing old html files
await Promise.all(parallelPromises); await Promise.all(parallelPromises);
} }

@ -2,7 +2,7 @@
const { News } = require(__dirname+'/../../db/') const { News } = require(__dirname+'/../../db/')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { buildNews } = require(__dirname+'/../../helpers/build.js') , buildQueue = require(__dirname+'/../../queue.js')
, linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js') , linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js')
, simpleMarkdown = require(__dirname+'/../../helpers/posting/markdown.js') , simpleMarkdown = require(__dirname+'/../../helpers/posting/markdown.js')
, escape = require(__dirname+'/../../helpers/posting/escape.js') , escape = require(__dirname+'/../../helpers/posting/escape.js')
@ -27,7 +27,10 @@ module.exports = async (req, res, next) => {
await News.insertOne(post); await News.insertOne(post);
await buildNews(); buildQueue.push({
'task': 'buildNews',
'options': {}
});
return res.render('message', { return res.render('message', {
'title': 'Success', 'title': 'Success',

@ -2,7 +2,7 @@
const { Boards, Posts, Accounts } = require(__dirname+'/../../db/') const { Boards, Posts, Accounts } = require(__dirname+'/../../db/')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { buildHomepage, buildCatalog, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js') , buildQueue = require(__dirname+'/../../queue.js')
, { remove } = require('fs-extra') , { remove } = require('fs-extra')
, deletePosts = require(__dirname+'/deletepost.js') , deletePosts = require(__dirname+'/deletepost.js')
, linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js') , linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js')
@ -101,15 +101,27 @@ module.exports = async (req, res, next) => {
for (let i = newMaxPage+1; i <= oldMaxPage; i++) { for (let i = newMaxPage+1; i <= oldMaxPage; i++) {
promises.push(remove(`${uploadDirectory}html/${req.params.board}/${i}.html`)); promises.push(remove(`${uploadDirectory}html/${req.params.board}/${i}.html`));
} }
//rebuild valid board pages for page numbers, and catalog for prunedthreads //rebuild all board pages for page nav numbers, and catalog
promises.push(buildBoardMultiple(res.locals.board, 1, newMaxPage)); buildQueue.push({
promises.push(buildCatalog(res.locals.board)); 'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': newMaxPage
}
});
buildQueue.push({
'task': 'buildCatalog',
'options': {
'board': res.locals.board,
}
});
} }
} }
if (newSettings.captchaMode !== oldSettings.captchaMode) { if (newSettings.captchaMode !== oldSettings.captchaMode) {
//TODO: only remove necessary pages here
promises.push(remove(`${uploadDirectory}html/${req.params.board}/`)); promises.push(remove(`${uploadDirectory}html/${req.params.board}/`));
//TODO: dont remove all pages, only remove some and add important pages to build queue here
} }
if (promises.length > 0) { if (promises.length > 0) {

@ -3,7 +3,7 @@
const { remove } = require('fs-extra') const { remove } = require('fs-extra')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { Boards } = require(__dirname+'/../../db/') , { Boards } = require(__dirname+'/../../db/')
, { buildBanners } = require(__dirname+'/../../helpers/build.js') , buildQueue = require(__dirname+'/../../queue.js');
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
@ -15,7 +15,7 @@ module.exports = async (req, res, next) => {
})); }));
//remove from db //remove from db
await Boards.removeBanners(req.params.board, req.body.checkedbanners); const amount = await Boards.removeBanners(req.params.board, req.body.checkedbanners);
//update res locals banners in memory //update res locals banners in memory
res.locals.board.banners = res.locals.board.banners.filter(banner => { res.locals.board.banners = res.locals.board.banners.filter(banner => {
@ -23,7 +23,12 @@ module.exports = async (req, res, next) => {
}); });
//rebuild public banners page //rebuild public banners page
await buildBanners(res.locals.board); buildQueue.push({
'task': 'buildBanners',
'options': {
'board': res.locals.board,
}
});
return res.render('message', { return res.render('message', {
'title': 'Success', 'title': 'Success',

@ -1,13 +1,16 @@
'use strict'; 'use strict';
const { News } = require(__dirname+'/../../db/') const { News } = require(__dirname+'/../../db/')
, { buildNews } = require(__dirname+'/../../helpers/build.js') , buildQueue = require(__dirname+'/../../queue.js')
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
await News.deleteMany(req.body.checkednews); await News.deleteMany(req.body.checkednews);
await buildNews(); buildQueue.push({
'task': 'buildNews',
'options': {}
});
return res.render('message', { return res.render('message', {
'title': 'Success', 'title': 'Success',

@ -27,7 +27,8 @@ const path = require('path')
, deletePosts = require(__dirname+'/deletepost.js') , deletePosts = require(__dirname+'/deletepost.js')
, spamCheck = require(__dirname+'/../../helpers/checks/spamcheck.js') , spamCheck = require(__dirname+'/../../helpers/checks/spamcheck.js')
, { postPasswordSecret } = require(__dirname+'/../../configs/main.json') , { postPasswordSecret } = require(__dirname+'/../../configs/main.json')
, { buildCatalog, buildThread, buildBoard, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js'); , buildQueue = require(__dirname+'/../../queue.js')
, { buildThread } = require(__dirname+'/../../helpers/build.js');
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
@ -395,23 +396,37 @@ module.exports = async (req, res, next) => {
} }
const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`; const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`;
console.log(`NEW POST -> ${successRedirect}`);
//build just the thread they need to see first and send them immediately //build just the thread they need to see first and send them immediately
await buildThread(data.thread || postId, res.locals.board); await buildThread({
'threadId': data.thread || postId,
'board': res.locals.board
});
res.redirect(successRedirect); res.redirect(successRedirect);
//now rebuild other pages //now add other pages to be built in background
const parallelPromises = [];
if (data.thread) { if (data.thread) {
//refersh pages //refersh pages
const threadPage = await Posts.getThreadPage(req.params.board, thread); const threadPage = await Posts.getThreadPage(req.params.board, thread);
if (data.email === 'sage' || thread.sage) { if (data.email === 'sage' || thread.sage) {
//refresh the page that the thread is on //refresh the page that the thread is on
parallelPromises.push(buildBoard(res.locals.board, threadPage)); buildQueue.push({
'task': 'buildBoard',
'options': {
'board': res.locals.board,
'page': threadPage
}
});
} else { } else {
//if not saged, it will bump so we should refresh any pages above it as well //if not saged, it will bump so we should refresh any pages above it as well
parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPage)); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': threadPage
}
});
} }
} else { } else {
//new thread, prunes any old threads before rebuilds //new thread, prunes any old threads before rebuilds
@ -419,13 +434,22 @@ module.exports = async (req, res, next) => {
if (prunedThreads.length > 0) { if (prunedThreads.length > 0) {
await deletePosts(prunedThreads, req.params.board); await deletePosts(prunedThreads, req.params.board);
} }
parallelPromises.push(buildBoardMultiple(res.locals.board, 1, Math.ceil(threadLimit/10))); buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': Math.ceil(threadLimit/10)
}
});
} }
//always rebuild catalog for post counts and ordering //always rebuild catalog for post counts and ordering
parallelPromises.push(buildCatalog(res.locals.board)); buildQueue.push({
'task': 'buildCatalog',
//finish building other pages 'options': {
await Promise.all(parallelPromises); 'board': res.locals.board,
}
});
} }

@ -8,7 +8,7 @@ const path = require('path')
, imageIdentify = require(__dirname+'/../../helpers/files/imageidentify.js') , imageIdentify = require(__dirname+'/../../helpers/files/imageidentify.js')
, deleteTempFiles = require(__dirname+'/../../helpers/files/deletetempfiles.js') , deleteTempFiles = require(__dirname+'/../../helpers/files/deletetempfiles.js')
, { Boards } = require(__dirname+'/../../db/') , { Boards } = require(__dirname+'/../../db/')
, { buildBanners } = require(__dirname+'/../../helpers/build.js') , buildQueue = require(__dirname+'/../../queue.js');
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
@ -79,8 +79,15 @@ module.exports = async (req, res, next) => {
//add banners to board in memory //add banners to board in memory
res.locals.board.banners = res.locals.board.banners.concat(filenames); res.locals.board.banners = res.locals.board.banners.concat(filenames);
// rebuild the public banners page if (filenames.length > 0) {
await buildBanners(res.locals.board); //add public banners page to build queue
buildQueue.push({
'task': 'buildBanners',
'options': {
'board': res.locals.board,
}
});
}
return res.render('message', { return res.render('message', {
'title': 'Success', 'title': 'Success',

@ -6,7 +6,7 @@ module.exports = async (req, res, next) => {
let html; let html;
try { try {
html = await buildBanners(res.locals.board); html = await buildBanners({ board: res.locals.board });
} catch (err) { } catch (err) {
return next(err); return next(err);
} }

@ -12,7 +12,11 @@ module.exports = async (req, res, next) => {
if (page > maxPage) { if (page > maxPage) {
return next(); return next();
} }
html = await buildBoard(res.locals.board, page, maxPage); html = await buildBoard({
board: res.locals.board,
page,
maxPage
});
} catch (err) { } catch (err) {
return next(err); return next(err);
} }

@ -6,7 +6,7 @@ module.exports = async (req, res, next) => {
let html; let html;
try { try {
html = await buildCatalog(res.locals.board); html = await buildCatalog({ board: res.locals.board });
} catch (err) { } catch (err) {
return next(err); return next(err);
} }

@ -19,7 +19,12 @@ module.exports = async (req, res, next) => {
if (!logs || logs.length === 0) { if (!logs || logs.length === 0) {
return next(); return next();
} }
html = await buildModLog(res.locals.board, startDate, endDate, logs); html = await buildModLog({
board: res.locals.board,
startDate,
endDate,
logs
});
} catch (err) { } catch (err) {
return next(err); return next(err);
} }

@ -6,7 +6,7 @@ module.exports = async (req, res, next) => {
let html; let html;
try { try {
html = await buildModLogList(res.locals.board); html = await buildModLogList({ board: res.locals.board });
} catch (err) { } catch (err) {
return next(err); return next(err);
} }

@ -6,7 +6,10 @@ module.exports = async (req, res, next) => {
let html; let html;
try { try {
html = await buildThread(res.locals.thread.postId, res.locals.board); html = await buildThread({
thredId: res.locals.thread.postId,
board: res.locals.board
});
} catch (err) { } catch (err) {
return next(err); return next(err);
} }

43
package-lock.json generated

@ -262,6 +262,14 @@
"resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.4.tgz", "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.4.tgz",
"integrity": "sha1-h3L80EGOPNLMFxVV1zAHQVBR9LI=" "integrity": "sha1-h3L80EGOPNLMFxVV1zAHQVBR9LI="
}, },
"@types/redis": {
"version": "2.8.13",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.13.tgz",
"integrity": "sha512-p86cm5P6DMotUqCS6odQRz0JJwc5QXZw9eyH0ALVIqmq12yqtex5ighWyGFHKxak9vaA/GF/Ilu0KZ0MuXXUbg==",
"requires": {
"@types/node": "*"
}
},
"abbrev": { "abbrev": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@ -1778,6 +1786,11 @@
"domelementtype": "1" "domelementtype": "1"
} }
}, },
"double-ended-queue": {
"version": "2.1.0-0",
"resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz",
"integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw="
},
"duplexify": { "duplexify": {
"version": "3.7.1", "version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@ -5638,6 +5651,26 @@
"resolve": "^1.1.6" "resolve": "^1.1.6"
} }
}, },
"redis": {
"version": "2.8.0",
"resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz",
"integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==",
"requires": {
"double-ended-queue": "^2.1.0-0",
"redis-commands": "^1.2.0",
"redis-parser": "^2.6.0"
}
},
"redis-commands": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz",
"integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg=="
},
"redis-parser": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz",
"integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs="
},
"regenerator-runtime": { "regenerator-runtime": {
"version": "0.11.1", "version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
@ -5842,6 +5875,16 @@
"resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz", "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz",
"integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w=" "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w="
}, },
"rsmq": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/rsmq/-/rsmq-0.11.0.tgz",
"integrity": "sha512-Uo+jQ85T3jbajAwoU7MGyIVRC1ytUDfMV+yjskQzIMEJDLWpcJmw+DlacM3EXwjwD9g4Q2yWdKuGXi8C2V8WIg==",
"requires": {
"@types/redis": "^2.8.0",
"lodash": "^4.17.11",
"redis": "^2.8.0"
}
},
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",

@ -32,6 +32,7 @@
"path": "^0.12.7", "path": "^0.12.7",
"pm2": "^3.5.1", "pm2": "^3.5.1",
"pug": "^2.0.4", "pug": "^2.0.4",
"rsmq": "^0.11.0",
"sanitize-html": "^1.20.1", "sanitize-html": "^1.20.1",
"saslprep": "^1.0.3" "saslprep": "^1.0.3"
}, },

@ -0,0 +1,32 @@
const RedisSMQ = require('rsmq')
, configs = require(__dirname+'/configs/main.json')
, rsmq = new RedisSMQ({ host: '127.0.0.1', port: 6379, ns: 'rsmq', password: configs.redisPassword })
, queuename = 'generate'
rsmq.createQueue({ qname: queuename }, (err) => {
if (err && err.name !== 'queueExists') {
return console.error(err);
}
});
module.exports.push = (data) => {
rsmq.sendMessage({ qname: queuename, message: JSON.stringify(data) }, (err) => {
if (err) {
return console.error(err);
}
//message enqueued successfully
});
}
/*
//was testing
setInterval(() => {
const data = {
task: 'buildCatalog',
options: {
'board': 'b'
}
}
module.exports.push(data);
}, 500);
*/

@ -7,24 +7,23 @@ process
const msTime = require(__dirname+'/helpers/mstime.js') const msTime = require(__dirname+'/helpers/mstime.js')
, deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js') , deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js')
, Mongo = require(__dirname+'/db/db.js') , Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js'); , Mutex = require(__dirname+'/mutex.js')
, buildQueue = require(__dirname+'/queue.js');
(async () => { (async () => {
await Mongo.connect(); await Mongo.connect();
await Mutex.connect(); await Mutex.connect();
const { buildHomepage } = require(__dirname+'/helpers/build.js') const Files = require(__dirname+'/db/files.js');
, Files = require(__dirname+'/db/files.js');
console.log('Starting schedules'); console.log('Starting schedules');
setInterval(async () => { // setInterval(async () => {
try { buildQueue.push({
await buildHomepage(); 'task': 'buildHomepage',
} catch (e) { 'options': {}
console.error(e); })
} // }, msTime.minute*5); //rebuild homepage for pph updates
}, msTime.minute*5); //rebuild homepage for pph updates
setInterval(async () => { setInterval(async () => {
try { try {

@ -0,0 +1,56 @@
'use strict';
process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const RedisSMQ = require('rsmq')
, configs = require(__dirname+'/configs/main.json')
, rsmq = new RedisSMQ({ host: '127.0.0.1', port: 6379, ns: 'rsmq', password: configs.redisPassword })
, queuename = 'generate'
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js');
let buildTasks = {}
, interval = 100;
(async () => {
await Mongo.connect();
await Mutex.connect();
buildTasks = require(__dirname+'/helpers/build.js');
rsmq.createQueue({ qname: queuename }, (err) => {
if (err && err.name !== 'queueExists') {
return console.error(err);
}
setTimeout(processQueueLoop, interval);
});
})();
function processQueueLoop() {
rsmq.receiveMessage({ qname: queuename }, async (err, resp) => {
if (err) {
return console.error(err);
}
if (resp.id) {
interval = 100; //keeps queue checking fast when there are tasks
const message = JSON.parse(resp.message);
await buildTasks[message.task](message.options);
rsmq.deleteMessage({ qname: queuename, id: resp.id }, (err) => {
if (err) {
return console.error(err);
}
//message deleted successfully
});
} else {
//max 2 sec poll time
if (interval < 2000) { //slow down queue when empty
interval += 100;
}
}
setTimeout(processQueueLoop, interval);
});
}
Loading…
Cancel
Save