locks for writing html files, also might need to add during dedupe to prevent file incs changing while pruning

5 years ago · 1e467fdc45
parent 686fe0d7b3
commit 1e467fdc45
8 changed files with 118 additions and 36 deletions
--- a/ecosystem.config.js
+++ b/ecosystem.config.js
@ -1,9 +1,24 @@
 module.exports = {
 	// Options reference: https://pm2.io/doc/en/runtime/reference/ecosystem-file/
 	apps : [{
+		name: 'lock-broker',
+		script: 'node_modules/live-mutex/dist/lm-start-server.js',
+		args: '--use-uds',
+		instances: 1,
+		autorestart: true,
+		watch: false,
+		max_memory_restart: '1G',
+		log_date_format: 'YYYY-MM-DD HH:mm:ss.SSS',
+		env: {
+			NODE_ENV: 'development'
+		},
+		env_production: {
+			NODE_ENV: 'production'
+		}
+	}, {
 		name: 'chan',
 		script: 'server.js',
-		instances: 2,
+		instances: 0, //0 = number of cpu cores
 		autorestart: true,
 		watch: false,
 		max_memory_restart: '1G',
--- a/helpers/build.js
+++ b/helpers/build.js
@ -8,18 +8,19 @@ const Mongo = require(__dirname+'/../db/db.js')
 module.exports = {

 	buildBanners: async(board) => {
-		const label = `${board._id}/banners.html`;
-		console.time(label);
+		const label = `/${board._id}/banners.html`;
+		const start = process.hrtime();
 		const html = render(label, 'banners.pug', {
 			board: board,
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

 	buildCatalog: async (board) => {
-		const label = `${board._id || board}/catalog.html`;
-		console.time(label);
+		const label = `/${board._id || board}/catalog.html`;
+		const start = process.hrtime();
 		if (!board._id) {
 			board = await Boards.findOne(board);
 		}
@ -28,13 +29,14 @@ module.exports = {
 			board,
 			threads,
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

 	buildThread: async (threadId, board) => {
-		const label = `${board._id || board}/thread/${threadId}.html`;
-		console.time(label);
+		const label = `/${board._id || board}/thread/${threadId}.html`;
+		const start = process.hrtime();
 		if (!board._id) {
 			board = await Boards.findOne(board);
 		}
@ -46,13 +48,14 @@ module.exports = {
 			board,
 			thread,
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

 	buildBoard: async (board, page, maxPage=null) => {
-		const label = `${board._id}/${page === 1 ? 'index' : page}.html`;
-		console.time(label);
+		const label = `/${board._id}/${page === 1 ? 'index' : page}.html`;
+		const start = process.hrtime();
 		const threads = await Posts.getRecent(board._id, page);
 		if (maxPage == null) {
 			maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10));
@ -64,14 +67,14 @@ module.exports = {
 			maxPage,
 			page,
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

 	//building multiple pages (for rebuilds)
 	buildBoardMultiple: async (board, startpage=1, endpage) => {
-		const label = 'multiple';
-		console.time(label);
+		const start = process.hrtime();
 		const maxPage = Math.min(Math.ceil((await Posts.getPages(board._id)) / 10), Math.ceil(board.settings.threadLimit/10));
 		if (endpage === 0) {
 			//deleted only/all posts, so only 1 page will remain
@ -82,7 +85,7 @@ module.exports = {
 		}
 		const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0
 		const threads = await Posts.getRecent(board._id, startpage, difference*10);
-		console.timeLog(label, `${board._id}/${startpage === 1 ? 'index' : startpage}.html => ${board._id}/${endpage === 1 ? 'index' : endpage}.html`)
+		const label = `/${board._id}/${startpage === 1 ? 'index' : startpage}.html => /${board._id}/${endpage === 1 ? 'index' : endpage}.html`;
 		const buildArray = [];
 		for (let i = startpage; i <= endpage; i++) {
 			let spliceStart = (i-1)*10;
@ -99,17 +102,19 @@ module.exports = {
 			);
 		}
 		await Promise.all(buildArray);
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 	},

 	buildNews: async () => {
 		const label = '/news.html';
-		console.time(label);
+		const start = process.hrtime();
 		const news = await News.find();
 		const html = render('news.html', 'news.pug', {
 			news
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

@ -124,7 +129,7 @@ module.exports = {
 		const month = ('0'+(startDate.getMonth()+1)).slice(-2);
 		const year = startDate.getFullYear();
 		const label = `/${board._id}/logs/${month}-${day}-${year}.html`;
-		console.time(label);
+		const start = process.hrtime();
 		if (!logs) {
 			logs = await Modlogs.findBetweenDate(board, startDate, endDate);
 		}
@ -134,25 +139,27 @@ module.exports = {
 			startDate,
 			endDate
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

 	buildModLogList: async (board) => {
 		const label = `/${board._id}/logs.html`;
-		console.time(label);
+		const start = process.hrtime();
 		const dates = await Modlogs.getDates(board);
 		const html = render(label, 'modloglist.pug', {
 			board,
 			dates
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

 	buildHomepage: async () => {
 		const label = '/index.html';
-		console.time(label);
+		const start = process.hrtime();
 		const [ activeUsers, postsPerHour ] = await Promise.all([
 			Posts.activeUsers(),
 			Posts.postsPerHour()
@ -214,7 +221,8 @@ module.exports = {
 			boards,
 			fileStats,
 		});
-		console.timeEnd(label);
+		const end = process.hrtime(start);
+		console.log(`${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`);
 		return html;
 	},

--- a/helpers/render.js
+++ b/helpers/render.js
@ -5,10 +5,13 @@ const { cacheTemplates, meta }= require(__dirname+'/../configs/main.json')
 	, pug = require('pug')
 	, path = require('path')
 	, uploadDirectory = require(__dirname+'/files/uploadDirectory.js')
+	, Mutex = require(__dirname+'/../mutex.js')
 	, templateDirectory = path.join(__dirname+'/../views/pages/');

 module.exports = async (htmlName, templateName, options) => {
 	const html = pug.renderFile(`${templateDirectory}${templateName}`, { ...options, cache: cacheTemplates, meta });
+	const { id, key } = await Mutex.acquire(htmlName);
 	await outputFile(`${uploadDirectory}html/${htmlName}`, html);
+	await Mutex.release(key, id);
 	return html;
 };
--- a/mutex.js
+++ b/mutex.js
@ -0,0 +1,5 @@
+'use strict';
+
+const { Client } = require('live-mutex');
+
+module.exports = new Client({ udsPath: process.env.HOME + '/.lmx/uds.sock' });
--- a/package-lock.json
+++ b/package-lock.json
@ -39,6 +39,19 @@
        }
      }
    },
+    "@oresoftware/json-stream-parser": {
+      "version": "0.0.113",
+      "resolved": "https://registry.npmjs.org/@oresoftware/json-stream-parser/-/json-stream-parser-0.0.113.tgz",
+      "integrity": "sha512-In7ufVanxBaCA7kxSRDtvkg5z/BInb5SneBhGPq+1UpRRqPUQ3KHECKKU3fdApHgPjnfbjGUQFCDaWvGVawraA=="
+    },
+    "@oresoftware/linked-queue": {
+      "version": "0.1.105",
+      "resolved": "https://registry.npmjs.org/@oresoftware/linked-queue/-/linked-queue-0.1.105.tgz",
+      "integrity": "sha512-XpYqg7BM39s9sF+1RSP7fpNNRsVNW847k+X8iiO26K939XeT4BHxG7ZBtwUVOVYI+T3JDjm1wcvGT2/pYH/7Jg==",
+      "requires": {
+        "chalk": "^2.4.2"
+      }
+    },
    "@pm2/agent": {
      "version": "0.5.26",
      "resolved": "https://registry.npmjs.org/@pm2/agent/-/agent-0.5.26.tgz",
@ -4077,6 +4090,18 @@
        "resolve": "^1.1.7"
      }
    },
+    "live-mutex": {
+      "version": "0.1.1066",
+      "resolved": "https://registry.npmjs.org/live-mutex/-/live-mutex-0.1.1066.tgz",
+      "integrity": "sha512-k61Ubi9lY+MatIUqCb14NFxun0BB7GUFjBwhxAR6/YzPf0h8zoKkZWwkPKUAZT4cWxLhX18ZNsZf/gNya+Dn2g==",
+      "requires": {
+        "@oresoftware/json-stream-parser": "0.0.113",
+        "@oresoftware/linked-queue": "0.1.105",
+        "chalk": "^2.4.2",
+        "tcp-ping": "^0.1.1",
+        "uuid": "^3.3.2"
+      }
+    },
    "load-json-file": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
@ -6334,6 +6359,11 @@
        }
      }
    },
+    "tcp-ping": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/tcp-ping/-/tcp-ping-0.1.1.tgz",
+      "integrity": "sha1-At1/QrW/fXy3jVt6rO+hVf2PfAw="
+    },
    "terser": {
      "version": "3.17.0",
      "resolved": "https://registry.npmjs.org/terser/-/terser-3.17.0.tgz",
--- a/package.json
+++ b/package.json
@ -25,6 +25,7 @@
    "gulp-less": "^4.0.1",
    "gulp-pug": "^4.0.1",
    "gulp-uglify-es": "^1.0.4",
+    "live-mutex": "^0.1.1066",
    "lodash": "^4.17.15",
    "lodash.mergewith": "^4.6.2",
    "mongodb": "^3.3.0",
--- a/schedules.js
+++ b/schedules.js
@ -6,17 +6,18 @@ process

 const msTime = require(__dirname+'/helpers/mstime.js')
 	, deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js')
-	, Mongo = require(__dirname+'/db/db.js');
+	, Mongo = require(__dirname+'/db/db.js')
+	, Mutex = require(__dirname+'/mutex.js');

 (async () => {

 	await Mongo.connect();
+	await Mutex.connect();
 	const { buildHomepage } = require(__dirname+'/helpers/build.js')
 		, Files = require(__dirname+'/db/files.js');

 	console.log('Starting schedules');

-await buildHomepage();
 	setInterval(async () => {
 		try {
 			await buildHomepage();
@ -35,13 +36,16 @@ await buildHomepage();

 	setInterval(async () => {
 		try {
-//todo: would need to lock the DB or at least disable posting very shortly for this pruning
-			const files = await Files.db.find({
+//todo: make this not a race condition, but it only happens daily so ¯\_(ツ)_/¯
+			const files = await Files.db.aggregate({
 				'count': {
-					'$lte': 0
+					'$lte': 1
 				}
 			}, {
-				'count': 0
+				'projection': {
+					'count': 0,
+					'size': 0
+				}
 			}).toArray().then(res => {
 				return res.map(x => x._id);
 			});
--- a/server.js
+++ b/server.js
@ -14,6 +14,7 @@ const express = require('express')
 	, configs = require(__dirname+'/configs/main.json')
 	, refererRegex = new RegExp(configs.refererRegex)
 	, Mongo = require(__dirname+'/db/db.js')
+	, Mutex = require(__dirname+'/mutex.js')
 	, { createHash } = require('crypto');

 (async () => {
@ -23,6 +24,10 @@ const express = require('express')
 	// let db connect
 	console.log('connecting to db');
 	await Mongo.connect();
+	const { Accounts } = require(__dirname+'/db/');
+
+	console.log('connecting to live mutex');
+	await Mutex.connect();

 	// disable useless express header
 	app.disable('x-powered-by');
@ -37,7 +42,10 @@ const express = require('express')
 	// session store
 	app.use(session({
 		secret: configs.sessionSecret,
-		store: new MongoStore({ db: Mongo.client.db('sessions') }),
+		store: new MongoStore({
+			db: Mongo.client.db('sessions'),
+			stringify: false
+		}),
 		resave: false,
 		saveUninitialized: false,
 		cookie: {
@ -50,11 +58,19 @@ const express = require('express')
 	//trust proxy for nginx
 	app.set('trust proxy', 1);

-	//referer header check
-	app.use((req, res, next) => {
+	//todo: refactor some of these middleware
+	app.use(async (req, res, next) => {
 		const ip = req.headers['x-real-ip'] || req.connection.remoteAddress
 		const ipHash = createHash('sha256').update(configs.ipHashSecret + ip).digest('base64');
 		res.locals.ip = ipHash;
+		if (req.session && req.session.authenticated === true) {
+			// keeping session updated incase user updated on global manage
+			const account = await Accounts.findOne(req.session.user.username);
+			req.session.user = {
+				'username': account._id,
+				'authLevel': account.authLevel
+			};
+		}
 		if (req.method !== 'POST') {
 			return next();
 		}
@ -98,7 +114,7 @@ const express = require('express')
 	// listen
 	const server = app.listen(configs.port, '127.0.0.1', () => {

-        console.log(`listening on port ${configs.port}`);
+		console.log(`listening on port ${configs.port}`);

 		//let PM2 know that this is ready (for graceful reloads)
 		if (typeof process.send === 'function') { //make sure we are a child process
@ -106,7 +122,7 @@ const express = require('express')
 			process.send('ready');
 		}

-    });
+	});

 	process.on('SIGINT', () => {