reference #256 unique message per board/thread option

strips quote, so starting post with quote isnt considered unique

configs/main.js.example

@@ -302,7 +302,8 @@ module.exports = {
 		codeTheme: 'ir-black',
 		sfw: false, //safe for work board
 		lockMode: 0, //board lock mode
-		fileR9KMode: 0, //r9k for files, 0=off, 1=per thread, 2=whole board
+		fileR9KMode: 0, //enfore unique files, 0=off, 1=per thread, 2=whole board
+		messageR9KMode: 0, //enforce unique message, 0=off, 1=per thread, 2=whole board
 		unlistedLocal: false, //board hidden from on-site board list and frontpage
 		unlistedWebring: false, //board hidden from webring
 		captchaMode: 0, //0=disabled, 1=for threads, 2=for all posts

db/posts.js

@@ -296,6 +296,25 @@ module.exports = {
 
 	},
 
+	checkExistingMessage: async (board, thread = null, hash) => {
+		const query = {
+			'board': board,
+			'messagehash': hash,
+		}
+		if (thread !== null) {
+			query['$or'] = [
+				{ 'thread': thread },
+				{ 'postId': thread },
+			]
+		}
+		const postWithExistingMessage = await db.findOne(query, {
+			'projection': {
+				'messagehash': 1,
+			}
+		});
+		return postWithExistingMessage;
+	},
+
 	checkExistingFiles: async (board, thread = null, hashes) => {
 		const query = {
 			'board': board,

helpers/paramconverter.js

@@ -5,7 +5,7 @@ const { ObjectId } = require(__dirname+'/../db/db.js')
 		'checkedreports', 'checkedbans', 'checkedbanners', 'checkedaccounts', 'countries']) //only these should be arrays, since express bodyparser can output arrays
 	, trimFields = ['tags', 'uri', 'moderators', 'filters', 'announcement', 'description', 'message',
 		'name', 'subject', 'email', 'postpassword', 'password', 'default_name', 'report_reason', 'ban_reason', 'log_message', 'custom_css'] //trim if we dont want filed with whitespace
-	, numberFields = ['filter_mode', 'lock_mode', 'file_r9k_mode', 'captcha_mode', 'tph_trigger', 'pph_trigger', 'trigger_action', 'bump_limit', 'reply_limit', 'move_to_thread',, 'postId',
+	, numberFields = ['filter_mode', 'lock_mode', 'message_r9k_mode', 'file_r9k_mode', 'captcha_mode', 'tph_trigger', 'pph_trigger', 'trigger_action', 'bump_limit', 'reply_limit', 'move_to_thread',, 'postId',
 		'max_files', 'thread_limit', 'thread', 'max_thread_message_length', 'max_reply_message_length', 'min_thread_message_length', 'min_reply_message_length', 'auth_level'] //convert these to numbers before they hit our routes
 	, banDurationRegex = /^(?<YEAR>[\d]+y)?(?<MONTH>[\d]+mo)?(?<WEEK>[\d]+w)?(?<DAY>[\d]+d)?(?<HOUR>[\d]+h)?(?<MINUTE>[\d]+m)?(?<SECOND>[\d]+s)?$/
 	, timeUtils = require(__dirname+'/timeutils.js')

helpers/posting/message.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const linkQuotes = require(__dirname+'/quotes.js')
+const quoteHandler = require(__dirname+'/quotes.js')
 	, { markdown } = require(__dirname+'/markdown.js')
 	, sanitizeOptions = require(__dirname+'/sanitizeoptions.js')
 	, sanitize = require('sanitize-html');
@@ -14,7 +14,7 @@ module.exports = async (inputMessage, boardName, threadId=null) => {
 	//markdown a post, link the quotes, sanitize and return message and quote arrays
 	if (message && message.length > 0) {
 		message = markdown(message);
-		const { quotedMessage, threadQuotes, crossQuotes } = await linkQuotes(boardName, message, threadId);
+		const { quotedMessage, threadQuotes, crossQuotes } = await quoteHandler.process(boardName, message, threadId);
 		message = quotedMessage;
 		quotes = threadQuotes;
 		crossquotes = crossQuotes;

helpers/posting/quotes.js

@@ -5,7 +5,11 @@ const Posts = require(__dirname+'/../../db/posts.js')
 	, quoteRegex = /&gt;&gt;(?<quotenum>\d+)/g
 	, crossQuoteRegex = /&gt;&gt;&gt;&#x2F;(?<board>\w+)(?:&#x2F;(?<quotenum>\d*))?/gm;
 
-module.exports = async (board, text, thread) => {
+module.exports = {
+
+	quoteRegex, crossQuoteRegex,
+
+	process: async (board, text, thread) => {
 
 		//get the matches
 		const quotes = text.match(quoteRegex);
@@ -116,4 +120,6 @@ module.exports = async (board, text, thread) => {
 
 		return { quotedMessage: text, threadQuotes: [...threadQuotes], crossQuotes: [...nonThreadQuotes] };
 
-}
+	},
+
+};

migrations/index.js

@@ -15,4 +15,5 @@ module.exports = {
 	'0.0.12': require(__dirname+'/migration-0.0.12.js'), //yotsuba b -> yotsuba-b
 	'0.0.13': require(__dirname+'/migration-0.0.13.js'), //add r9k mode (files)
 	'0.0.14': require(__dirname+'/migration-0.0.14.js'), //add option for disable .onion file posts to board settings
+	'0.0.15': require(__dirname+'/migration-0.0.15.js'), //messages r9k option
 }

migrations/migration-0.0.15.js

@@ -0,0 +1,12 @@
+'use strict';
+
+module.exports = async(db, redis) => {
+	console.log('Adding message r9k option to boards db');
+	await db.collection('boards').updateMany({}, {
+		'$set': {
+			'settings.messageR9KMode': 0,
+		}
+	});
+	console.log('Cleared boards cache');
+	await redis.deletePattern('board:*');
+};

models/forms/changeboardsettings.js

@@ -107,6 +107,7 @@ module.exports = async (req, res, next) => {
 		'maxThreadMessageLength': numberSetting(req.body.max_thread_message_length, oldSettings.maxThreadMessageLength),
 		'maxReplyMessageLength': numberSetting(req.body.max_reply_message_length, oldSettings.maxReplyMessageLength),
 		'lockMode': numberSetting(req.body.lock_mode, oldSettings.lockMode),
+		'messageR9KMode': numberSetting(req.body.message_r9k_mode, oldSettings.messageR9KMode),
 		'fileR9KMode': numberSetting(req.body.file_r9k_mode, oldSettings.fileR9KMode),
 		'filterMode': numberSetting(req.body.filter_mode, oldSettings.filterMode),
 		'filterBanDuration': numberSetting(req.body.ban_duration, oldSettings.filterBanDuration),

models/forms/deletepost.js

@@ -4,7 +4,7 @@ const uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.
 	, { remove } = require('fs-extra')
 	, Mongo = require(__dirname+'/../../db/db.js')
 	, { Posts, Files } = require(__dirname+'/../../db/')
-	, linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js')
+	, quoteHandler = require(__dirname+'/../../helpers/posting/quotes.js')
 	, { markdown } = require(__dirname+'/../../helpers/posting/markdown.js')
 	, { pruneImmediately } = require(__dirname+'/../../configs/main.js')
 	, pruneFiles = require(__dirname+'/../../schedules/prune.js')
@@ -119,7 +119,7 @@ module.exports = async (posts, board, all=false) => {
 				if (post.nomarkup && post.nomarkup.length > 0) { //is this check even necessary? how would it have a quote with no message
 					//redo the markup
 					let message = markdown(post.nomarkup);
-					const { quotedMessage, threadQuotes, crossQuotes } = await linkQuotes(post.board, message, post.thread);
+					const { quotedMessage, threadQuotes, crossQuotes } = await quoteHandler.process(post.board, message, post.thread);
 					message = sanitize(quotedMessage, sanitizeOptions.after);
 					bulkWrites.push({
 						'updateOne': {

models/forms/editpost.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { Posts, Bans, Modlogs } = require(__dirname+'/../../db/')
+	, { createHash } = require('crypto')
 	, Mongo = require(__dirname+'/../../db/db.js')
 	, getTripCode = require(__dirname+'/../../helpers/posting/tripcode.js')
 	, { prepareMarkdown } = require(__dirname+'/../../helpers/posting/markdown.js')
@@ -73,6 +74,12 @@ todo: handle some more situations
 		}
 	}
 
+	//message hash
+	let messageHash = null;
+	if (req.body.message && req.body.message.length > 0) {
+		const noQuoteMessage = req.body.message.replace(/>>\d+/g, '').replace(/>>>\/\w+(\/\d*)?/gm, '').trim();
+		messageHash = createHash('sha256').update(noQuoteMessage).digest('base64');
+	}
 	//new name, trip and cap
 	const { name, tripcode, capcode } = await nameHandler(req.body.name, res.locals.permLevel,
 		board.settings, board.owner, res.locals.user ? res.locals.user.username : null);
@@ -130,6 +137,7 @@ todo: handle some more situations
 			},
 			nomarkup,
 			message,
+			'messagehash': messageHash || null,
 			quotes,
 			crossquotes,
 			name,

models/forms/makepost.js

@@ -50,7 +50,7 @@ module.exports = async (req, res, next) => {
 	const { filterBanDuration, filterMode, filters, blockedCountries, resetTrigger,
 			maxFiles, sageOnlyEmail, forceAnon, replyLimit, disableReplySubject,
 			threadLimit, ids, userPostSpoiler, pphTrigger, tphTrigger, triggerAction,
-			captchaMode, lockMode, allowedFileTypes, flags, fileR9KMode } = res.locals.board.settings;
+			captchaMode, lockMode, allowedFileTypes, flags, fileR9KMode, messageR9KMode } = res.locals.board.settings;
 	if (res.locals.permLevel >= 4
 		&& res.locals.country
 		&& blockedCountries.includes(res.locals.country.code)) {
@@ -158,10 +158,29 @@ module.exports = async (req, res, next) => {
 		}
 
 	}
+
+	//for r9k messages. usually i wouldnt process these if its not enabled e.g. flags and IDs but in this case I think its necessary
+	let messageHash = null;
+	if (req.body.message && req.body.message.length > 0) {
+		const noQuoteMessage = req.body.message.replace(/>>\d+/g, '').replace(/>>>\/\w+(\/\d*)?/gm, '').trim();
+		messageHash = createHash('sha256').update(noQuoteMessage).digest('base64');
+		if (res.locals.permLevel >= 4 && (req.body.thread && messageR9KMode === 1) || messageR9KMode === 2) {
+			const postWithExistingMessage = await Posts.checkExistingMessage(res.locals.board._id, (messageR9KMode === 2 ? null : req.body.thread), messageHash);
+			if (postWithExistingMessage != null) {
+				await deleteTempFiles(req).catch(e => console.error);
+				return dynamicResponse(req, res, 409, 'message', {
+					'title': 'Conflict',
+					'message': `Messages must be unique ${messageR9KMode === 1 ? 'in this thread' : 'on this board'}. Your message is not unique.`,
+					'redirect': redirect
+				});
+			}
+		}
+	}
+
 	let files = [];
 	// if we got a file
 	if (res.locals.numFiles > 0) {
-		if ((req.body.thread && fileR9KMode === 1) || fileR9KMode === 2) {
+		if (res.locals.permLevel >= 4 && (req.body.thread && fileR9KMode === 1) || fileR9KMode === 2) {
 			const filesHashes = req.files.file.map(f => f.sha256);
 			const postWithExistingFiles = await Posts.checkExistingFiles(res.locals.board._id, (fileR9KMode === 2 ? null : req.body.thread), filesHashes);
 			if (postWithExistingFiles != null) {
@@ -401,7 +420,7 @@ module.exports = async (req, res, next) => {
 	const nomarkup = prepareMarkdown(req.body.message, true);
 	const { message, quotes, crossquotes } = await messageHandler(nomarkup, req.params.board, req.body.thread);
 
-	//build post data for db
+	//build post data for db. for some reason all the property names are lower case :^)
 	const data = {
 		'date': new Date(),
 		name,
@@ -411,6 +430,7 @@ module.exports = async (req, res, next) => {
 		capcode,
 		subject,
 		'message': message || null,
+		'messagehash': messageHash || null,
 		'nomarkup': nomarkup || null,
 		'thread': req.body.thread || null,
 		password,

models/forms/moveposts.js

@@ -3,7 +3,7 @@
 const uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
 	, { remove } = require('fs-extra')
 	, { Posts } = require(__dirname+'/../../db/')
-	, linkQuotes = require(__dirname+'/../../helpers/posting/quotes.js')
+	, quoteHandler = require(__dirname+'/../../helpers/posting/quotes.js')
 	, { markdown } = require(__dirname+'/../../helpers/posting/markdown.js')
 	, sanitize = require('sanitize-html')
 	, sanitizeOptions = require(__dirname+'/../../helpers/posting/sanitizeoptions.js');
@@ -118,7 +118,7 @@ module.exports = async (req, res) => {
 			if (post.nomarkup && post.nomarkup.length > 0) {
 				//redo the markup
 				let message = markdown(post.nomarkup);
-				let { quotedMessage, threadQuotes, crossQuotes } = await linkQuotes(post.board, message, post.thread); // req.body.move_to_thread);
+				let { quotedMessage, threadQuotes, crossQuotes } = await quoteHandler.process(post.board, message, post.thread); // req.body.move_to_thread);
 //console.log(quotedMessage, threadQuotes, crossQuotes)
 				message = sanitize(quotedMessage, sanitizeOptions.after);
 				bulkWrites.push({

package.json

@@ -1,7 +1,7 @@
 {
   "name": "jschan",
   "version": "0.0.1",
-  "migrateVersion": "0.0.14",
+  "migrateVersion": "0.0.15",
   "description": "",
   "main": "server.js",
   "dependencies": {

remarkup.js

@@ -10,7 +10,7 @@ const Mongo = require(__dirname+'/db/db.js');
 
 	await Mongo.connect();
 	const { Posts } = require(__dirname+'/db/')
-		, linkQuotes = require(__dirname+'/helpers/posting/quotes.js')
+		, quoteHandler = require(__dirname+'/helpers/posting/quotes.js')
 		, { markdown } = require(__dirname+'/helpers/posting/markdown.js')
 		, sanitizeOptions = require(__dirname+'/helpers/posting/sanitizeoptions.js')
 		, sanitize = require('sanitize-html');
@@ -18,7 +18,7 @@ const Mongo = require(__dirname+'/db/db.js');
 	const posts = await Posts.db.find({/*query here*/}).toArray();
 	await Promise.all(posts.map(async (post) => {
 		let message = markdown(post.nomarkup);
-		const { quotedMessage, threadQuotes, crossQuotes } = await linkQuotes(post.board, message, null);
+		const { quotedMessage, threadQuotes, crossQuotes } = await quoteHandler.process(post.board, message, null);
 		message = sanitize(quotedMessage, sanitizeOptions.after);
 		console.log(post.postId, message.substring(0,10)+'...');
 		return Posts.db.updateOne({board:post.board, postId:post.postId}, {$set:{message:message}});

views/pages/managesettings.pug

@@ -162,11 +162,17 @@ block content
 							option(value='1', selected=board.settings.lockMode === 1) Lock thread creation
 							option(value='2', selected=board.settings.lockMode === 2) Lock board
 					.row
-						.label Unique Files
+						.label Enforce Unique Files
 						select(name='file_r9k_mode')
 							option(value='0', selected=board.settings.fileR9KMode === 0) Off
 							option(value='1', selected=board.settings.fileR9KMode === 1) Per Thread
 							option(value='2', selected=board.settings.fileR9KMode === 2) Board Wide
+					.row
+						.label Enforce Unique Messages
+						select(name='message_r9k_mode')
+							option(value='0', selected=board.settings.messageR9KMode === 0) Off
+							option(value='1', selected=board.settings.messageR9KMode === 1) Per Thread
+							option(value='2', selected=board.settings.messageR9KMode === 2) Board Wide
 					.row
 						.label Unlist locally
 						label.postform-style.ph-5