use a queue with workers for generating static pages in background that arent immediately needed

merge-requests/208/head
fatchan 5 years ago
parent 52b37fb34b
commit 7868e1c5d3
  1. 5
      README.md
  2. 1
      configs/main.json.example
  3. 16
      ecosystem.config.js
  4. 128
      helpers/build.js
  5. 88
      models/forms/actionhandler.js
  6. 7
      models/forms/addnews.js
  7. 22
      models/forms/changeboardsettings.js
  8. 11
      models/forms/deletebanners.js
  9. 7
      models/forms/deletenews.js
  10. 66
      models/forms/makepost.js
  11. 13
      models/forms/uploadbanners.js
  12. 2
      models/pages/banners.js
  13. 6
      models/pages/board.js
  14. 2
      models/pages/catalog.js
  15. 7
      models/pages/modlog.js
  16. 2
      models/pages/modloglist.js
  17. 5
      models/pages/thread.js
  18. 43
      package-lock.json
  19. 1
      package.json
  20. 32
      queue.js
  21. 19
      schedules.js
  22. 56
      worker.js

@ -40,9 +40,10 @@ Please note:
##### Requirements
- Linux (most likely could work elsewhere, but why?)
- Node.js (to run the app)
- MongoDB (database, duh)
- Redis (queues, and eventually for caching and mutex/locking)
- Nginx (handle https, serve static content and html)
- Certbot/letsencrypt (for https cert)
- MongoDB (database, duh)
- Imagemagick (thumbnailing images)
- Ffmpeg (thumbnailing videos)
- Bcrypt (account password hashes)
@ -55,6 +56,8 @@ $ sudo apt-get install bcrypt nginx ffmpeg imagemagick
[Install](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-debian/#install-mongodb-community-edition-on-debian) and [configure auth for mongodb](https://medium.com/mongoaudit/how-to-enable-authentication-on-mongodb-b9e8a924efac). This is to avoid out of date verisons in debian repos.
[Install and configure](https://www.digitalocean.com/community/tutorials/how-to-install-and-secure-redis-on-debian-9) Redis.
Install nodejs. You can use [node version manager](https://github.com/nvm-sh/nvm) (nvm) to help with this.
Once you have nvm, install the LTS version of nodejs (currently 10.x).
```bash

@ -5,6 +5,7 @@
"tripcodeSecret": "long random string",
"ipHashSecret": "long random string",
"postPasswordSecret": "long random string",
"redisPassword": "long random string",
"cacheTemplates": true,
"refererCheck": false,
"refererRegex": "^https?:\\/\\/(?:www\\.)?domain\\.com\\/",

@ -15,10 +15,24 @@ module.exports = {
env_production: {
NODE_ENV: 'production'
}
}, {
name: 'build-worker',
script: 'worker.js',
instances: 1, //could increase if building is getting backed up
autorestart: true,
watch: false,
max_memory_restart: '1G',
log_date_format: 'YYYY-MM-DD HH:mm:ss.SSS',
env: {
NODE_ENV: 'development'
},
env_production: {
NODE_ENV: 'production'
}
}, {
name: 'chan',
script: 'server.js',
instances: 0, //0 = number of cpu cores
instances: 0, // 0 = number of cpu cores
autorestart: true,
watch: false,
max_memory_restart: '1G',

@ -3,98 +3,95 @@
const Mongo = require(__dirname+'/../db/db.js')
, msTime = require(__dirname+'/mstime.js')
, { Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/')
, render = require(__dirname+'/render.js');
, render = require(__dirname+'/render.js')
, timeDiffString = (label, end) => `${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`;
module.exports = {
buildBanners: async(board) => {
const label = `/${board._id}/banners.html`;
buildBanners: async (options) => {
const label = `/${options.board._id}/banners.html`;
const start = process.hrtime();
const html = render(label, 'banners.pug', {
board: board,
});
const html = render(label, 'banners.pug', options);
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
buildCatalog: async (board) => {
const label = `/${board._id || board}/catalog.html`;
buildCatalog: async (options) => {
const label = `/${options.board._id || options.board}/catalog.html`;
const start = process.hrtime();
if (!board._id) {
board = await Boards.findOne(board);
if (!options.board._id) {
options.board = await Boards.findOne(options.board);
}
const threads = await Posts.getCatalog(board._id);
const threads = await Posts.getCatalog(options.board._id);
const html = render(label, 'catalog.pug', {
board,
...options,
threads,
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
buildThread: async (threadId, board) => {
const label = `/${board._id || board}/thread/${threadId}.html`;
buildThread: async (options) => {
const label = `/${options.board._id || options.board}/thread/${options.threadId}.html`;
const start = process.hrtime();
if (!board._id) {
board = await Boards.findOne(board);
if (!options.board._id) {
options.board = await Boards.findOne(options.board);
}
const thread = await Posts.getThread(board._id, threadId)
const thread = await Posts.getThread(options.board._id, options.threadId);
if (!thread) {
return; //this thread may have been an OP that was deleted
}
const html = render(label, 'thread.pug', {
board,
...options,
thread,
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
buildBoard: async (board, page, maxPage=null) => {
const label = `/${board._id}/${page === 1 ? 'index' : page}.html`;
buildBoard: async (options) => {
const label = `/${options.board._id}/${options.page === 1 ? 'index' : options.page}.html`;
const start = process.hrtime();
const threads = await Posts.getRecent(board._id, page);
if (maxPage == null) {
maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10));
const threads = await Posts.getRecent(options.board._id, options.page);
if (!options.maxPage) {
options.maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10));
}
const html = render(label, 'board.pug', {
board,
...options,
threads,
maxPage,
page,
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
//building multiple pages (for rebuilds)
buildBoardMultiple: async (board, startpage=1, endpage) => {
buildBoardMultiple: async (options) => {
const start = process.hrtime();
const maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10));
if (endpage === 0) {
const maxPage = Math.min(Math.ceil((await Posts.getPages(options.board._id)) / 10), Math.ceil(options.board.settings.threadLimit/10));
if (options.endpage === 0) {
//deleted only/all posts, so only 1 page will remain
endpage = 1;
} else if (maxPage < endpage) {
options.endpage = 1;
} else if (maxPage < options.endpage) {
//else just build up to the max page if it is greater than input page number
endpage = maxPage
options.endpage = maxPage
}
const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0
const threads = await Posts.getRecent(board._id, startpage, difference*10);
const label = `/${board._id}/${startpage === 1 ? 'index' : startpage}.html => /${board._id}/${endpage === 1 ? 'index' : endpage}.html`;
const difference = options.endpage-options.startpage + 1; //+1 because for single pagemust be > 0
const threads = await Posts.getRecent(options.board._id, options.startpage, difference*10);
const label = `/${options.board._id}/${options.startpage === 1 ? 'index' : options.startpage}.html => /${options.board._id}/${options.endpage === 1 ? 'index' : options.endpage}.html`;
const buildArray = [];
for (let i = startpage; i <= endpage; i++) {
for (let i = options.startpage; i <= options.endpage; i++) {
let spliceStart = (i-1)*10;
if (spliceStart > 0) {
spliceStart = spliceStart - 1;
}
buildArray.push(
render(`${board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', {
board,
render(`${options.board._id}/${i === 1 ? 'index' : i}.html`, 'board.pug', {
board: options.board,
threads: threads.splice(0,10),
maxPage,
page: i,
@ -103,7 +100,7 @@ module.exports = {
}
await Promise.all(buildArray);
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
},
buildNews: async () => {
@ -114,46 +111,43 @@ module.exports = {
news
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
buildModLog: async (board, startDate, endDate, logs) => {
if (!startDate || !endDate) {
startDate = new Date(); //this is being built by action handler so will always be current date
endDate = new Date(startDate.getTime());
startDate.setHours(0,0,0,0);
endDate.setHours(23,59,59,999);
buildModLog: async (options) => {
if (!options.startDate || !options.endDate) {
options.startDate = new Date(); //this is being built by action handler so will always be current date
options.endDate = new Date(options.startDate.getTime());
options.startDate.setHours(0,0,0,0);
options.endDate.setHours(23,59,59,999);
}
const day = ('0'+startDate.getDate()).slice(-2);
const month = ('0'+(startDate.getMonth()+1)).slice(-2);
const year = startDate.getFullYear();
const label = `/${board._id}/logs/${month}-${day}-${year}.html`;
const day = ('0'+options.startDate.getDate()).slice(-2);
const month = ('0'+(options.startDate.getMonth()+1)).slice(-2);
const year = options.startDate.getFullYear();
const label = `/${options.board._id}/logs/${month}-${day}-${year}.html`;
const start = process.hrtime();
if (!logs) {
logs = await Modlogs.findBetweenDate(board, startDate, endDate);
if (!options.logs) {
options.logs = await Modlogs.findBetweenDate(options.board, options.startDate, options.endDate);
}
const html = render(label, 'modlog.pug', {
board,
logs,
startDate,
endDate
...options
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
buildModLogList: async (board) => {
const label = `/${board._id}/logs.html`;
buildModLogList: async (options) => {
const label = `/${options.board._id}/logs.html`;
const start = process.hrtime();
const dates = await Modlogs.getDates(board);
const dates = await Modlogs.getDates(options.board);
const html = render(label, 'modloglist.pug', {
board,
board: options.board,
dates
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},
@ -222,7 +216,7 @@ module.exports = {
fileStats,
});
const end = process.hrtime(start);
console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
console.log(timeDiffString(label, end));
return html;
},

@ -14,7 +14,7 @@ const { Posts, Boards, Modlogs } = require(__dirname+'/../../db/')
, dismissReports = require(__dirname+'/dismissreport.js')
, { remove } = require('fs-extra')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { buildModLog, buildModLogList, buildCatalog, buildThread, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js')
, buildQueue = require(__dirname+'/../../queue.js')
, { postPasswordSecret } = require(__dirname+'/../../configs/main.json')
, { createHash, timingSafeEqual } = require('crypto');
@ -245,8 +245,18 @@ module.exports = async (req, res, next) => {
await Modlogs.insertMany(modlogDocuments);
for (let i = 0; i < threadBoards.length; i++) {
const board = buildBoards[threadBoards[i]];
parallelPromises.push(buildModLog(board));
parallelPromises.push(buildModLogList(board));
buildQueue.push({
'task': 'buildModLog',
'options': {
'board': res.locals.board,
}
});
buildQueue.push({
'task': 'buildModLogList',
'options': {
'board': res.locals.board,
}
});
}
}
}
@ -315,7 +325,13 @@ module.exports = async (req, res, next) => {
const board = buildBoards[boardName];
//rebuild impacted threads
for (let j = 0; j < boardThreadMap[boardName].threads.length; j++) {
parallelPromises.push(buildThread(boardThreadMap[boardName].threads[j], board));
buildQueue.push({
'task': 'buildThread',
'options': {
'threadId': boardThreadMap[boardName].threads[j],
'board': board,
}
});
}
//refersh any pages affected
const afterPages = Math.ceil((await Posts.getPages(boardName)) / 10);
@ -329,26 +345,68 @@ module.exports = async (req, res, next) => {
parallelPromises.push(remove(`${uploadDirectory}html/${boardName}/${k}.html`));
}
}
parallelPromises.push(buildBoardMultiple(board, 1, afterPages));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': 1,
'endpage': afterPages,
}
});
} else {
//number of pages did not change, only possibly building existing pages
const threadPageOldest = await Posts.getThreadPage(boardName, bounds.oldest);
const threadPageNewest = bounds.oldest.postId === bounds.newest.postId ? threadPageOldest : await Posts.getThreadPage(boardName, bounds.newest);
if (req.body.delete || req.body.delete_ip_board || req.body.delete_ip_global) {
if (!boardThreadMap[boardName].directThreads) {
//onyl deleting posts from threads, so thread order wont change, thus we dont delete all pages after
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest));
//only deleting posts from threads, so thread order wont change, thus we dont delete all pages after
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': threadPageOldest,
}
});
} else {
//deleting threads, so we delete all pages after
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, afterPages));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': afterPages,
}
});
}
} else if (req.body.sticky) { //else if -- if deleting, other actions are not executed/irrelevant
//rebuild current and newer pages
parallelPromises.push(buildBoardMultiple(board, 1, threadPageOldest));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': 1,
'endpage': threadPageOldest,
}
});
} else if (req.body.lock || req.body.sage || req.body.cyclic || req.body.unlink_file) {
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': threadPageOldest,
}
});
} else if (req.body.spoiler || req.body.ban || req.body.global_ban) {
parallelPromises.push(buildBoardMultiple(board, threadPageNewest, threadPageOldest));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': board,
'startpage': threadPageNewest,
'endpage': afterPages,
}
});
if (!boardThreadMap[boardName].directThreads) {
catalogRebuild = false;
//these actions dont affect the catalog tile since not on an OP and dont change reply/image counts
@ -357,12 +415,18 @@ module.exports = async (req, res, next) => {
}
if (catalogRebuild) {
//the actions will affect the catalog, so we better rebuild it
parallelPromises.push(buildCatalog(board));
buildQueue.push({
'task': 'buildCatalog',
'options': {
'board': board,
}
});
}
}
}
if (parallelPromises.length > 0) {
//since queue changes, this just removing old html files
await Promise.all(parallelPromises);
}

@ -2,7 +2,7 @@
const { News } = require(__dirname+'/../../db/')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { buildNews } = require(__dirname+'/../../helpers/build.js')
, buildQueue = require(__dirname+'/../../queue.js')
, linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js')
, simpleMarkdown = require(__dirname+'/../../helpers/posting/markdown.js')
, escape = require(__dirname+'/../../helpers/posting/escape.js')
@ -27,7 +27,10 @@ module.exports = async (req, res, next) => {
await News.insertOne(post);
await buildNews();
buildQueue.push({
'task': 'buildNews',
'options': {}
});
return res.render('message', {
'title': 'Success',

@ -2,7 +2,7 @@
const { Boards, Posts, Accounts } = require(__dirname+'/../../db/')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { buildHomepage, buildCatalog, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js')
, buildQueue = require(__dirname+'/../../queue.js')
, { remove } = require('fs-extra')
, deletePosts = require(__dirname+'/deletepost.js')
, linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js')
@ -101,15 +101,27 @@ module.exports = async (req, res, next) => {
for (let i = newMaxPage+1; i <= oldMaxPage; i++) {
promises.push(remove(`${uploadDirectory}html/${req.params.board}/${i}.html`));
}
//rebuild valid board pages for page numbers, and catalog for prunedthreads
promises.push(buildBoardMultiple(res.locals.board, 1, newMaxPage));
promises.push(buildCatalog(res.locals.board));
//rebuild all board pages for page nav numbers, and catalog
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': newMaxPage
}
});
buildQueue.push({
'task': 'buildCatalog',
'options': {
'board': res.locals.board,
}
});
}
}
if (newSettings.captchaMode !== oldSettings.captchaMode) {
//TODO: only remove necessary pages here
promises.push(remove(`${uploadDirectory}html/${req.params.board}/`));
//TODO: dont remove all pages, only remove some and add important pages to build queue here
}
if (promises.length > 0) {

@ -3,7 +3,7 @@
const { remove } = require('fs-extra')
, uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
, { Boards } = require(__dirname+'/../../db/')
, { buildBanners } = require(__dirname+'/../../helpers/build.js')
, buildQueue = require(__dirname+'/../../queue.js');
module.exports = async (req, res, next) => {
@ -15,7 +15,7 @@ module.exports = async (req, res, next) => {
}));
//remove from db
await Boards.removeBanners(req.params.board, req.body.checkedbanners);
const amount = await Boards.removeBanners(req.params.board, req.body.checkedbanners);
//update res locals banners in memory
res.locals.board.banners = res.locals.board.banners.filter(banner => {
@ -23,7 +23,12 @@ module.exports = async (req, res, next) => {
});
//rebuild public banners page
await buildBanners(res.locals.board);
buildQueue.push({
'task': 'buildBanners',
'options': {
'board': res.locals.board,
}
});
return res.render('message', {
'title': 'Success',

@ -1,13 +1,16 @@
'use strict';
const { News } = require(__dirname+'/../../db/')
, { buildNews } = require(__dirname+'/../../helpers/build.js')
, buildQueue = require(__dirname+'/../../queue.js')
module.exports = async (req, res, next) => {
await News.deleteMany(req.body.checkednews);
await buildNews();
buildQueue.push({
'task': 'buildNews',
'options': {}
});
return res.render('message', {
'title': 'Success',

@ -27,20 +27,21 @@ const path = require('path')
, deletePosts = require(__dirname+'/deletepost.js')
, spamCheck = require(__dirname+'/../../helpers/checks/spamcheck.js')
, { postPasswordSecret } = require(__dirname+'/../../configs/main.json')
, { buildCatalog, buildThread, buildBoard, buildBoardMultiple } = require(__dirname+'/../../helpers/build.js');
, buildQueue = require(__dirname+'/../../queue.js')
, { buildThread } = require(__dirname+'/../../helpers/build.js');
module.exports = async (req, res, next) => {
//spam/flood check
const flood = await spamCheck(req, res);
if (flood) {
deleteTempFiles(req).catch(e => console.error);
return res.status(429).render('message', {
'title': 'Flood detected',
'message': 'Please wait before making another post, or a post similar to another user',
'redirect': `/${req.params.board}${req.body.thread ? '/thread/' + req.body.thread + '.html' : ''}`
});
}
const flood = await spamCheck(req, res);
if (flood) {
deleteTempFiles(req).catch(e => console.error);
return res.status(429).render('message', {
'title': 'Flood detected',
'message': 'Please wait before making another post, or a post similar to another user',
'redirect': `/${req.params.board}${req.body.thread ? '/thread/' + req.body.thread + '.html' : ''}`
});
}
// check if this is responding to an existing thread
let redirect = `/${req.params.board}/`
@ -395,23 +396,37 @@ module.exports = async (req, res, next) => {
}
const successRedirect = `/${req.params.board}/thread/${req.body.thread || postId}.html#${postId}`;
console.log(`NEW POST -> ${successRedirect}`);
//build just the thread they need to see first and send them immediately
await buildThread(data.thread || postId, res.locals.board);
await buildThread({
'threadId': data.thread || postId,
'board': res.locals.board
});
res.redirect(successRedirect);
//now rebuild other pages
const parallelPromises = [];
//now add other pages to be built in background
if (data.thread) {
//refersh pages
const threadPage = await Posts.getThreadPage(req.params.board, thread);
if (data.email === 'sage' || thread.sage) {
//refresh the page that the thread is on
parallelPromises.push(buildBoard(res.locals.board, threadPage));
buildQueue.push({
'task': 'buildBoard',
'options': {
'board': res.locals.board,
'page': threadPage
}
});
} else {
//if not saged, it will bump so we should refresh any pages above it as well
parallelPromises.push(buildBoardMultiple(res.locals.board, 1, threadPage));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': threadPage
}
});
}
} else {
//new thread, prunes any old threads before rebuilds
@ -419,13 +434,22 @@ module.exports = async (req, res, next) => {
if (prunedThreads.length > 0) {
await deletePosts(prunedThreads, req.params.board);
}
parallelPromises.push(buildBoardMultiple(res.locals.board, 1, Math.ceil(threadLimit/10)));
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': Math.ceil(threadLimit/10)
}
});
}
//always rebuild catalog for post counts and ordering
parallelPromises.push(buildCatalog(res.locals.board));
//finish building other pages
await Promise.all(parallelPromises);
buildQueue.push({
'task': 'buildCatalog',
'options': {
'board': res.locals.board,
}
});
}

@ -8,7 +8,7 @@ const path = require('path')
, imageIdentify = require(__dirname+'/../../helpers/files/imageidentify.js')
, deleteTempFiles = require(__dirname+'/../../helpers/files/deletetempfiles.js')
, { Boards } = require(__dirname+'/../../db/')
, { buildBanners } = require(__dirname+'/../../helpers/build.js')
, buildQueue = require(__dirname+'/../../queue.js');
module.exports = async (req, res, next) => {
@ -79,8 +79,15 @@ module.exports = async (req, res, next) => {
//add banners to board in memory
res.locals.board.banners = res.locals.board.banners.concat(filenames);
// rebuild the public banners page
await buildBanners(res.locals.board);
if (filenames.length > 0) {
//add public banners page to build queue
buildQueue.push({
'task': 'buildBanners',
'options': {
'board': res.locals.board,
}
});
}
return res.render('message', {
'title': 'Success',

@ -6,7 +6,7 @@ module.exports = async (req, res, next) => {
let html;
try {
html = await buildBanners(res.locals.board);
html = await buildBanners({ board: res.locals.board });
} catch (err) {
return next(err);
}

@ -12,7 +12,11 @@ module.exports = async (req, res, next) => {
if (page > maxPage) {
return next();
}
html = await buildBoard(res.locals.board, page, maxPage);
html = await buildBoard({
board: res.locals.board,
page,
maxPage
});
} catch (err) {
return next(err);
}

@ -6,7 +6,7 @@ module.exports = async (req, res, next) => {
let html;
try {
html = await buildCatalog(res.locals.board);
html = await buildCatalog({ board: res.locals.board });
} catch (err) {
return next(err);
}

@ -19,7 +19,12 @@ module.exports = async (req, res, next) => {
if (!logs || logs.length === 0) {
return next();
}
html = await buildModLog(res.locals.board, startDate, endDate, logs);
html = await buildModLog({
board: res.locals.board,
startDate,
endDate,
logs
});
} catch (err) {
return next(err);
}

@ -6,7 +6,7 @@ module.exports = async (req, res, next) => {
let html;
try {
html = await buildModLogList(res.locals.board);
html = await buildModLogList({ board: res.locals.board });
} catch (err) {
return next(err);
}

@ -6,7 +6,10 @@ module.exports = async (req, res, next) => {
let html;
try {
html = await buildThread(res.locals.thread.postId, res.locals.board);
html = await buildThread({
thredId: res.locals.thread.postId,
board: res.locals.board
});
} catch (err) {
return next(err);
}

43
package-lock.json generated

@ -262,6 +262,14 @@
"resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.4.tgz",
"integrity": "sha1-h3L80EGOPNLMFxVV1zAHQVBR9LI="
},
"@types/redis": {
"version": "2.8.13",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.13.tgz",
"integrity": "sha512-p86cm5P6DMotUqCS6odQRz0JJwc5QXZw9eyH0ALVIqmq12yqtex5ighWyGFHKxak9vaA/GF/Ilu0KZ0MuXXUbg==",
"requires": {
"@types/node": "*"
}
},
"abbrev": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@ -1778,6 +1786,11 @@
"domelementtype": "1"
}
},
"double-ended-queue": {
"version": "2.1.0-0",
"resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz",
"integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw="
},
"duplexify": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@ -5638,6 +5651,26 @@
"resolve": "^1.1.6"
}
},
"redis": {
"version": "2.8.0",
"resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz",
"integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==",
"requires": {
"double-ended-queue": "^2.1.0-0",
"redis-commands": "^1.2.0",
"redis-parser": "^2.6.0"
}
},
"redis-commands": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz",
"integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg=="
},
"redis-parser": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz",
"integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs="
},
"regenerator-runtime": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
@ -5842,6 +5875,16 @@
"resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz",
"integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w="
},
"rsmq": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/rsmq/-/rsmq-0.11.0.tgz",
"integrity": "sha512-Uo+jQ85T3jbajAwoU7MGyIVRC1ytUDfMV+yjskQzIMEJDLWpcJmw+DlacM3EXwjwD9g4Q2yWdKuGXi8C2V8WIg==",
"requires": {
"@types/redis": "^2.8.0",
"lodash": "^4.17.11",
"redis": "^2.8.0"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",

@ -32,6 +32,7 @@
"path": "^0.12.7",
"pm2": "^3.5.1",
"pug": "^2.0.4",
"rsmq": "^0.11.0",
"sanitize-html": "^1.20.1",
"saslprep": "^1.0.3"
},

@ -0,0 +1,32 @@
const RedisSMQ = require('rsmq')
, configs = require(__dirname+'/configs/main.json')
, rsmq = new RedisSMQ({ host: '127.0.0.1', port: 6379, ns: 'rsmq', password: configs.redisPassword })
, queuename = 'generate'
rsmq.createQueue({ qname: queuename }, (err) => {
if (err && err.name !== 'queueExists') {
return console.error(err);
}
});
module.exports.push = (data) => {
rsmq.sendMessage({ qname: queuename, message: JSON.stringify(data) }, (err) => {
if (err) {
return console.error(err);
}
//message enqueued successfully
});
}
/*
//was testing
setInterval(() => {
const data = {
task: 'buildCatalog',
options: {
'board': 'b'
}
}
module.exports.push(data);
}, 500);
*/

@ -7,24 +7,23 @@ process
const msTime = require(__dirname+'/helpers/mstime.js')
, deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js')
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js');
, Mutex = require(__dirname+'/mutex.js')
, buildQueue = require(__dirname+'/queue.js');
(async () => {
await Mongo.connect();
await Mutex.connect();
const { buildHomepage } = require(__dirname+'/helpers/build.js')
, Files = require(__dirname+'/db/files.js');
const Files = require(__dirname+'/db/files.js');
console.log('Starting schedules');
setInterval(async () => {
try {
await buildHomepage();
} catch (e) {
console.error(e);
}
}, msTime.minute*5); //rebuild homepage for pph updates
// setInterval(async () => {
buildQueue.push({
'task': 'buildHomepage',
'options': {}
})
// }, msTime.minute*5); //rebuild homepage for pph updates
setInterval(async () => {
try {

@ -0,0 +1,56 @@
'use strict';
process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const RedisSMQ = require('rsmq')
, configs = require(__dirname+'/configs/main.json')
, rsmq = new RedisSMQ({ host: '127.0.0.1', port: 6379, ns: 'rsmq', password: configs.redisPassword })
, queuename = 'generate'
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js');
let buildTasks = {}
, interval = 100;
(async () => {
await Mongo.connect();
await Mutex.connect();
buildTasks = require(__dirname+'/helpers/build.js');
rsmq.createQueue({ qname: queuename }, (err) => {
if (err && err.name !== 'queueExists') {
return console.error(err);
}
setTimeout(processQueueLoop, interval);
});
})();
function processQueueLoop() {
rsmq.receiveMessage({ qname: queuename }, async (err, resp) => {
if (err) {
return console.error(err);
}
if (resp.id) {
interval = 100; //keeps queue checking fast when there are tasks
const message = JSON.parse(resp.message);
await buildTasks[message.task](message.options);
rsmq.deleteMessage({ qname: queuename, id: resp.id }, (err) => {
if (err) {
return console.error(err);
}
//message deleted successfully
});
} else {
//max 2 sec poll time
if (interval < 2000) { //slow down queue when empty
interval += 100;
}
}
setTimeout(processQueueLoop, interval);
});
}
Loading…
Cancel
Save