Merge branch 'new-dev'

CHANGELOG.md

@@ -83,3 +83,16 @@
   - Some visual tweaks (file "(u)" being on newline, "x" -> "×")
   - Bugfixes
 
+### 0.2.0
+  - From now on, versioning = major.minor.patch. significant changes = major, feature updates = minor, bugfixes/small stuff = patch.
+  - Update instructions about nginx changes when upgrading.
+  - Add an endpoint for getting the csrf token separately from html pages. See API docs for more details.
+  - Add post "marking" so moved/deleted posts info is sent over websocket. Frontend will handle them. Deleted threads and moved OPs will now also disconnect the socket and remove the post form.
+  - Block bypasses are now locked to where they were created (anonymizer or clearnet) to prevent some forms of 'smuggling'. This will be improved further in upcoming releases.
+  - Code highlighting now supports all highlight.js languages when explicitly specified. The whitelist now only applies to auto-detection, as originally intended.
+  - Quotes for post references in modlog now have the proper quote class, and will show when hovered like any other quote.
+  - Bugfixes
+  - [jschan-docs](http://fatchan.gitgud.site/jschan-docs/):
+    - API docs improvements, now includes csrf token, posting, post actions (and mod variants), and more. It should be enough documentation for somebody to write a mobile app integration.
+  - [globalafk](https://gitgud.io/fatchan/globalafk/) (my fork):
+    - On android with termux, tapping the notification will open the post (in mod view) and the notification has new shortcut buttons to quickly delete, delete+ban or delete+global ban.

INSTALLATION.md

@@ -116,18 +116,28 @@ To enable the proxy, tick "Use Socks Proxy" in global management settings and se
 ## Updating
 
 ```bash
-#stop the jschan backend
+#first, stop the jschan backend
 $ pm2 stop ecosystem.config.js
+
 #pull the latest changes
 $ git pull
+
 #install dependencies again in case any have updated or changed
 $ npm install
+
+#check if anything nginx related changed between the old and new verison, e.g.
+$ git diff v0.1.5 v0.1.6 configs/nginx
+#If you use a completely standard jschan nginx, run configs/nginx/nginx.sh again.
+#Otherwise, update your nginx config with the necessary changes.
+
 #run the gulp migrate task. this will update things such as your database schema.
 $ gulp migrate
 #run the default gulp task to update, scripts, css, icons, images and delete old html
 $ gulp
+
 #start the backend again
 $ pm2 restart ecosystem.config.js --env production
+
 #if something breaks, check and read the logs, they will help figure out what went wrong
 $ pm2 logs
 ```

configs/nginx/snippets/error_pages.conf

@@ -4,22 +4,22 @@ error_page 502 /502.html;
 error_page 503 /503.html;
 error_page 504 /504.html;
 location = /404.html {
-        root /path/to/jschan/static/html;
-        internal;
+	root /path/to/jschan/static/html;
+	internal;
 }
 location = /500.html {
-        root /path/to/jschan/static/html;
-        internal;
+	root /path/to/jschan/static/html;
+	internal;
 }
 location = /502.html {
-        root /path/to/jschan/static/html;
-        internal;
+	root /path/to/jschan/static/html;
+	internal;
 }
 location = /503.html {
-        root /path/to/jschan/static/html;
-        internal;
+	root /path/to/jschan/static/html;
+	internal;
 }
 location = /504.html {
-        root /path/to/jschan/static/html;
-        internal;
+	root /path/to/jschan/static/html;
+	internal;
 }

configs/nginx/snippets/jschan_common_routes.conf

@@ -29,23 +29,23 @@ location /captcha {
 }
 
 # authed, no cache pages
-location ~* ^/((\w+/manage/.*|globalmanage/(reports|bans|recent|boards|globallogs|news|accounts|settings))|account|create)\.(html|json)$ {
-        expires 0;
-        try_files /dev/null @backend-private;
+location ~* ^/((\w+/manage/.*|globalmanage/(reports|bans|recent|boards|globallogs|news|accounts|settings))|account|create|csrf)\.(html|json)$ {
+	expires 0;
+	try_files /dev/null @backend-private;
 }
 
 # public html
 location ~* \.html$ {
-        expires 0;
-        root /path/to/jschan/static/html;
-        try_files $uri @backend;
+	expires 0;
+	root /path/to/jschan/static/html;
+	try_files $uri @backend;
 }
 
 # public json
 location ~* \.json$ {
-        expires 0;
-        root /path/to/jschan/static/json;
-        try_files $uri @backend;
+	expires 0;
+	root /path/to/jschan/static/json;
+	try_files $uri @backend;
 }
 
 # CSS

configs/template.js.example

@@ -354,6 +354,7 @@ module.exports = {
 		notificationsEnabled: false, //show notifications for new posts in a thread
 		notificationsYousOnly: true, //only show notifications for posts that quote you
 		showYous: true, //show (You) next to name on your posts, and quotes linking to your post
+		hideDeletedPostContent: false, //hide the message and files from marked deleted posts from websocket
 	},
 
 	//default board settings when a board is created

controllers/forms.js

@@ -51,14 +51,12 @@ router.post('/board/:board/transfer', useSession, sessionRefresh, csrf, Boards.e
 router.post('/board/:board/settings', geoAndTor, torPreBypassCheck, processIp, useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), boardSettingsController.paramConverter, boardSettingsController.controller);
 router.post('/board/:board/editbans', useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(3), editBansController.paramConverter, editBansController.controller); //edit bans
 router.post('/board/:board/deleteboard', useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(config.get.deleteBoardPermLevel), deleteBoardController.controller); //delete board
+
 //board crud banners, flags, assets, custompages
 router.post('/board/:board/addbanners', useSession, sessionRefresh, fileMiddlewares.banner, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), numFiles, uploadBannersController.controller); //add banners
 router.post('/board/:board/deletebanners', useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), deleteBannersController.paramConverter, deleteBannersController.controller); //delete banners
-
-///--- wip
 router.post('/board/:board/addassets', useSession, sessionRefresh, fileMiddlewares.asset, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), numFiles, addAssetsController.controller); //add assets
 router.post('/board/:board/deleteassets', useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), deleteAssetsController.paramConverter, deleteAssetsController.controller); //delete assets
-
 router.post('/board/:board/addflags', useSession, sessionRefresh, fileMiddlewares.flag, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), numFiles, addFlagsController.controller); //add flags
 router.post('/board/:board/deleteflags', useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), deleteFlagsController.paramConverter, deleteFlagsController.controller); //delete flags
 router.post('/board/:board/addcustompages', useSession, sessionRefresh, csrf, Boards.exists, calcPerms, isLoggedIn, hasPerms(2), addCustomPageController.paramConverter, addCustomPageController.controller); //add custom pages

controllers/forms/makepost.js

@@ -30,7 +30,7 @@ module.exports = {
 			{ result: (lengthBody(req.body.message, 1) && res.locals.numFiles === 0), expected: false, error: 'Posts must include a message or file' },
 			{ result: (res.locals.anonymizer && (disableAnonymizerFilePosting || res.locals.board.settings.disableAnonymizerFilePosting)
 				&& res.locals.numFiles > 0), expected: false, error: `Posting files through anonymizers has been disabled ${disableAnonymizerFilePosting ? 'globally' : 'on this board'}` },
-			{ result: res.locals.numFiles > res.locals.board.settings.maxFiles, blocking: true, permLevel: 1, expected: true, error: `Too many files. Max files per post ${res.locals.board.settings.maxFiles < globalLimits.postFiles.max ? 'on this board ' : ''}is ${res.locals.board.settings.maxFiles}` },
+			{ result: res.locals.numFiles > res.locals.board.settings.maxFiles, blocking: true, expected: false, error: `Too many files. Max files per post ${res.locals.board.settings.maxFiles < globalLimits.postFiles.max ? 'on this board ' : ''}is ${res.locals.board.settings.maxFiles}` },
 			{ result: (lengthBody(req.body.subject, 1) && (!existsBody(req.body.thread)
 				&& res.locals.board.settings.forceThreadSubject)), expected: false, error: 'Threads must include a subject' },
 			{ result: lengthBody(req.body.message, 1) && (!existsBody(req.body.thread)

controllers/pages.js

@@ -22,7 +22,7 @@ const express  = require('express')
 		globalManageRecent, globalManageAccounts, globalManageNews, globalManageLogs } = require(__dirname+'/../models/pages/globalmanage/')
 	, { changePassword, blockBypass, home, register, login, create,
 		board, catalog, banners, randombanner, news, captchaPage, overboard, overboardCatalog,
-		captcha, thread, modlog, modloglist, account, boardlist, customPage } = require(__dirname+'/../models/pages/')
+		captcha, thread, modlog, modloglist, account, boardlist, customPage, csrfPage } = require(__dirname+'/../models/pages/')
 	, threadParamConverter = paramConverter({ processThreadIdParam: true })
 	, logParamConverter = paramConverter({ processDateParam: true })
 	, newsParamConverter = paramConverter({ objectIdParams: ['newsid'] })
@@ -90,5 +90,6 @@ router.get('/login.html', login);
 router.get('/register.html', register);
 router.get('/changepassword.html', changePassword);
 router.get('/create.html', useSession, sessionRefresh, isLoggedIn, create); //create new board
+router.get('/csrf.json', useSession, sessionRefresh, isLoggedIn, csrf, csrfPage); //just the token, for 3rd party stuff posting
 
 module.exports = router;

db/bypass.js

@@ -8,10 +8,11 @@ module.exports = {
 
 	db,
 
-	checkBypass: (id) => {
+	checkBypass: (id, anonymizer=false) => {
 		const { blockBypass } = config.get;
 		return db.findOneAndUpdate({
 			'_id': id,
+			'anonymizer': anonymizer,
 			'uses': {
 				'$lte': blockBypass.expireAfterUses
 			}
@@ -22,10 +23,11 @@ module.exports = {
 		}).then(r => r.value);
 	},
 
-	getBypass: () => {
+	getBypass: (anonymizer=false) => {
 		const { blockBypass } = config.get;
 		return db.insertOne({
 			'uses': 0,
+			'anonymizer': anonymizer,
 			'expireAt': new Date(Date.now() + blockBypass.expireAfterTime)
 		});
 	},

gulp/res/css/style.css

@@ -1026,6 +1026,16 @@ input:invalid, textarea:invalid {
 	background: none;
 }
 
+.post-container.marked::after {
+	content: attr(data-mark) " ";
+	font-weight: bold;
+	color: var(--title-color);
+}
+
+.post-container.marked .post-data, .post-container.marked .post-info {
+	opacity: 0.75
+}
+
 .anchor:target + .post-container,
 .anchor:target + .catalog-tile,
 .post-container.highlighted,

gulp/res/js/csstoggles.js

@@ -43,6 +43,7 @@ class CssToggle {
 
 //define the css
 const hidePostStubsCss = `.post-container.hidden, .catalog-tile.hidden { display: none;margin-top: -1.5em;height: 0; }`;
+const hideDeletedPostContentCss = `.post-container.marked[data-mark="Deleted"] .post-data { display: none; }`;
 const hideThumbnailsCss = `.file-thumb, .catalog-thumb { visibility: hidden !important; }`;
 const hideRecursiveCss = `.op.hidden ~ .anchor, .op.hidden ~ .post-container { display: none; }`;
 const heightUnlimitCss = `img, video { max-height: unset; }`;
@@ -60,5 +61,6 @@ new CssToggle('hidethumbnails-setting', 'hidethumbnails', settings.hideThumbnail
 new CssToggle('noncolorids-setting', 'noncolorids', settings.nonColorIds, nonColorIdsCss);
 new CssToggle('alwaysshowspoilers-setting', 'alwaysshowspoilers', settings.alwaysShowSpoilers, alwaysShowSpoilersCss);
 new CssToggle('hidepoststubs-setting', 'hidepoststubs', settings.hidePostStubs, hidePostStubsCss);
+new CssToggle('hidedeletedpostcontent-setting', 'hidedeletedpostcontent', settings.hideDeletedPostContent, hideDeletedPostContentCss);
 new CssToggle('smoothscrolling-setting', 'smoothscrolling', settings.smoothScrolling, smoothScrollingCss);
 new CssToggle('threadwatcher-setting', 'threadwatcher', settings.threadWatcher, threadWatcherCss);

gulp/res/js/filters.js

@@ -77,7 +77,7 @@ const togglePostsHidden = (posts, state, single) => {
 		} else {
 			elem.classList['add']('hidden');
 		}
-			elem.querySelector('.postmenu').children[0].textContent = (showing ? 'Hide' : 'Show');
+		elem.querySelector('.postmenu').children[0].textContent = (showing ? 'Hide' : 'Show');
 	}
 };
 

gulp/res/js/importexport.js

@@ -2,7 +2,7 @@ window.addEventListener('settingsReady', () => {
 
 	const settingNames = ['volume','loop','imageloadingbars','live','scroll','localtime','relative','24hour','notifications','hiddenimages', 'threadwatcher'
 		,'notification-yous-only','yous-setting','filters1','name','theme','codetheme','customcss','disableboardcss','hiderecursive', 'watchlist'
-		,'heightlimit','crispimages','hidethumbnails','noncolorids','alwaysshowspoilers','hidepoststubs','smoothscrolling'];
+		,'heightlimit','crispimages','hidethumbnails','noncolorids','alwaysshowspoilers','hidedeletedpostcontent','hidepoststubs','smoothscrolling'];
 
 	const importExportText = document.getElementById('import-export-setting');
 

gulp/res/js/live.js

@@ -26,6 +26,39 @@ window.addEventListener('settingsReady', function(event) { //after domcontentloa
 		lastPostIds[board] = Math.max((lastPostIds[board] || 0), postId);
 	}
 
+	//add text before post-info to show posts deleted, moved, etc
+	const markPost = (data) => {
+		console.log('got mark post message', data);
+		const anchor = document.getElementById(data.postId);
+		const postContainer = anchor.nextSibling;
+		postContainer.classList.add('marked');
+		postContainer.setAttribute('data-mark', data.mark);
+		//handle any special cases for different marks
+		switch (data.type) {
+			case "delete":
+			case "move":
+				if (postContainer.classList.contains('op')) {
+					//moved or delete OPs then apply to whole thread
+					const postContainers = document.getElementsByClassName('post-container');
+					Array.from(postContainers).forEach(e => {
+						e.classList.add('marked')
+						e.setAttribute('data-mark', data.mark);
+					});
+					//remove new reply buttons and postform
+					document.getElementById('postform').remove();
+					const postButtons = document.getElementsByClassName('post-button');
+					Array.from(postButtons).forEach(e => e.remove());
+					//and disconnect socket
+					if (socket.connected === true) {
+						socket.disconnect();
+					}
+				}
+				break;
+			default:
+				//nothing special
+		}
+	};
+
 	const newPost = (data) => {
 		//insert at end of thread, but insert at top for globalmanage
 		console.log('got new post', data);
@@ -236,6 +269,7 @@ window.addEventListener('settingsReady', function(event) { //after domcontentloa
 				enableLive();
 			});
 			socket.on('newPost', newPost);
+			socket.on('markPost', markPost);
 		} else {
 			//websocket not supported, update with polling to api
 			updateButton.removeAttribute('style');

gulp/res/js/watchlist.js

@@ -45,7 +45,7 @@ class ThreadWatcher {
 
 	//refresh all the threads in the watchlist map
 	refresh() {
-		console.log('refreshing watchlist')
+		this.watchListMap.size > 0 && console.log('refreshing watchlist');
 		for (let t of this.watchListMap.entries()) {
 			const [board, postId] = t[0].split('-');
 			const data = t[1];

helpers/checks/blockbypass.js

@@ -37,7 +37,7 @@ module.exports = async (req, res, next) => {
 	if (bypassId && bypassId.length === 24) {
 		try {
 			const bypassMongoId = ObjectId(bypassId);
-			bypass = await Bypass.checkBypass(bypassMongoId);
+			bypass = await Bypass.checkBypass(bypassMongoId, res.locals.anonymizer);
 			res.locals.blockBypass = true;
 		} catch (err) {
 			return next(err);
@@ -53,7 +53,7 @@ module.exports = async (req, res, next) => {
 
 	if (res.locals.solvedCaptcha) {
 		//they dont have a valid bypass, but just solved board captcha, so give them a new one
-		const newBypass = await Bypass.getBypass();
+		const newBypass = await Bypass.getBypass(res.locals.anonymizer);
 		const newBypassId = newBypass.insertedId;
 		res.locals.blockBypass = true;
 		res.cookie('bypassid', newBypassId.toString(), {

helpers/checks/torprebypass.js

@@ -50,7 +50,7 @@ module.exports = async (req, res, next) => {
 			&& !blockBypass.forceAnonymizers //AND its not forced for anonymizers
 			&& !bypassId)) { //AND they dont already have one,
 		//then give the user a bypass id
-		const newBypass = await Bypass.getBypass();
+		const newBypass = await Bypass.getBypass(res.locals.anonymizer);
 		const newBypassId = newBypass.insertedId;
 		bypassId = newBypassId.toString();
 		res.locals.preFetchedBypassId = bypassId;

helpers/filemiddlewares.js

@@ -3,7 +3,7 @@
 const { debugLogs } = require(__dirname+'/../configs/secrets.js')
 	, dynamicResponse = require(__dirname+'/dynamic.js')
 	, { addCallback } = require(__dirname+'/../redis.js')
-	, upload = require('express-fileupload')
+	, upload = require('@fatchan/express-fileupload')
 	, fileHandlers = {}
 	, fileSizeLimitFunction = (req, res, next) => {
 		return dynamicResponse(req, res, 413, 'message', {
@@ -25,9 +25,10 @@ const { debugLogs } = require(__dirname+'/../configs/secrets.js')
 			const fileSizeLimit = globalLimits[`${fileType}FilesSize`];
 			const fileNumLimit = globalLimits[`${fileType}Files`];
 			const fileNumLimitFunction = (req, res, next) => {
+				const isPostform = req.path.endsWith('/post') || req.path.endsWith('/modpost');
 				return dynamicResponse(req, res, 400, 'message', {
 					'title': 'Too many files',
-					'message': (req.path.endsWith('/post') && res.locals.board) ? `Max files per post ${res.locals.board.settings.maxFiles < globalLimits.postFiles.max ? 'on this board ' : ''}is ${res.locals.board.settings.maxFiles}`
+					'message': (isPostform && res.locals.board) ? `Max files per post ${res.locals.board.settings.maxFiles < globalLimits.postFiles.max ? 'on this board ' : ''}is ${res.locals.board.settings.maxFiles}`
 						: `Max files per request is ${fileNumLimit.max}`,
 					'redirect': req.headers.referer
 				});

helpers/posting/markdown.js

@@ -16,7 +16,8 @@ const greentextRegex = /^>((?!>\d+|>>/\w+(/\d*)?|>>#
 	, splitRegex = /\[code\]([\s\S]+?)\[\/code\]/gm
 	, trimNewlineRegex = /^(\s*\r?\n)*/g
 	, escape = require(__dirname+'/escape.js')
-	, { highlight, highlightAuto } = require('highlight.js')
+	, { highlight, highlightAuto, listLanguages } = require('highlight.js')
+	, validLanguages = listLanguages() //precompute
 	, { addCallback } = require(__dirname+'/../../redis.js')
 	, config = require(__dirname+'/../../config.js')
 	, diceroll = require(__dirname+'/diceroll.js')
@@ -90,18 +91,18 @@ module.exports = {
 			lang = matches.groups.language.toLowerCase();
 		}
 		if (!lang) {
+			//no language specified, try automatic syntax highlighting
 			const { language, relevance, value } = highlightAuto(trimFix, highlightOptions.languageSubset);
 			if (relevance > highlightOptions.threshold) {
 				return `<span class='code hljs'><small>possible language: ${language}, relevance: ${relevance}</small>\n${value}</span>`;
 			}
-		} else if (lang !== 'plain' && highlightOptions.languageSubset.includes(lang)) {
-			if (lang === 'aa') {
-				return `<span class='aa'>${escape(matches.groups.code)}</span>`;
-			} else {
-				const { value } = highlight(trimFix, { language: lang, ignoreIllegals: true });
-				return `<span class='code hljs'><small>language: ${lang}</small>\n${value}</span>`;
-			}
+		} else if (lang === 'aa') {
+			return `<span class='aa'>${escape(matches.groups.code)}</span>`;
+		} else if (validLanguages.includes(lang)) {
+			const { value } = highlight(trimFix, { language: lang, ignoreIllegals: true });
+			return `<span class='code hljs'><small>language: ${lang}</small>\n${value}</span>`;
 		}
+		//else, auto highlight relevance threshold was too low, lang was not a valid language, or lang was 'plain'
 		return `<span class='code'>${escape(trimFix)}</span>`;
 	},
 

helpers/schema.js

@@ -54,7 +54,7 @@ module.exports = {
 
 	//the opposite kinda, check if the data includes any of the values in the array
 	arrayInBody: (filters, data) => {
-		return filters.some(filter => data.includes(filter));
+		return data && filters.some(filter => data.includes(filter));
 	},
 
 	//check the actual schema

helpers/settingsdiff.js

@@ -9,11 +9,11 @@ function getDotProp(obj, prop) {
 }
 
 function includeChildren(template, prop, tasks) {
-	Object.keys(getDotProp(template, prop))
-	.reduce((a, x) => {
-		a[`${prop}.${x}`] = tasks;
-		return a;
-	}, {});
+	return Object.keys(getDotProp(template, prop))
+		.reduce((a, x) => {
+			a[`${prop}.${x}`] = tasks;
+			return a;
+		}, {});
 }
 
 function compareSettings(entries, oldObject, newObject, maxSetSize) {

migrations/0.2.0.js

@@ -0,0 +1,12 @@
+'use strict';
+
+module.exports = async(db, redis) => {
+	console.log('Adding option to hide deleted post content to frontend script settings');
+	await db.collection('globalsettings').updateOne({ _id: 'globalsettings' }, {
+		'$set': {
+			'frontendScriptDefault.hideDeletedPostContent': false,
+		},
+	});
+	console.log('Clearing globalsettings cache');
+	await redis.deletePattern('globalsettings');
+};

models/forms/blockbypass.js

@@ -8,7 +8,7 @@ const { Bypass } = require(__dirname+'/../../db/')
 module.exports = async (req, res, next) => {
 
 	const { secureCookies, blockBypass } = config.get;
-	const bypass = await Bypass.getBypass();
+	const bypass = await Bypass.getBypass(res.locals.anonymizer);
 	const bypassId = bypass.insertedId;
 	res.locals.blockBypass = true;
 

models/forms/changeglobalsettings.js

@@ -165,6 +165,7 @@ module.exports = async (req, res, next) => {
 			notificationsEnabled: booleanSetting(req.body.frontend_script_default_notifications_embed, oldSettings.frontendScriptDefault.notificationsEnabled),
 			notificationsYousOnly: booleanSetting(req.body.frontend_script_default_notifications_yous_only, oldSettings.frontendScriptDefault.notificationsYousOnly),
 			showYous: booleanSetting(req.body.frontend_script_default_show_yous, oldSettings.frontendScriptDefault.showYous),
+			hideDeletedPostContent: booleanSetting(req.body.frontend_script_default_hide_deleted_post_content, oldSettings.frontendScriptDefault.hideDeletedPostContent),
 		},
 		animatedGifThumbnails: booleanSetting(req.body.animated_gif_thumbnails, oldSettings.animatedGifThumbnails),
 		audioThumbnails: booleanSetting(req.body.audio_thumbnails, oldSettings.audioThumbnails),

models/forms/deletepost.js

@@ -4,6 +4,7 @@ const uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.
 	, { remove } = require('fs-extra')
 	, Mongo = require(__dirname+'/../../db/db.js')
 	, { Posts, Files } = require(__dirname+'/../../db/')
+	, Socketio = require(__dirname+'/../../socketio.js')
 	, quoteHandler = require(__dirname+'/../../helpers/posting/quotes.js')
 	, { markdown } = require(__dirname+'/../../helpers/posting/markdown.js')
 	, config = require(__dirname+'/../../config.js')
@@ -18,13 +19,15 @@ module.exports = async (posts, board, all=false) => {
 	//filter to threads
 	const threads = posts.filter(x => x.thread == null);
 
-	if (threads.length > 0) {
-		//delete the html/json for threads
-		await Promise.all(threads.map(thread => {
-			remove(`${uploadDirectory}/html/${thread.board}/thread/${thread.postId}.html`)
-			remove(`${uploadDirectory}/json/${thread.board}/thread/${thread.postId}.json`)
-		}));
-	}
+	//emits not including the fetched posts from next block because those are based on threads being selected
+	//and we dont need to send delete message for every reply in a thread when the OP gets deleted.
+	const deleteEmits = posts.reduce((acc, post) => {
+		acc.push({
+			room: `${post.board}-${post.thread || post.postId}`,
+			postId: post.postId,
+		});
+		return acc;
+	}, []);
 
 	//get posts from all threads
 	let threadPosts = []
@@ -35,7 +38,7 @@ module.exports = async (posts, board, all=false) => {
 			threadPosts = await Posts.getMultipleThreadPosts(board, threadPostIds);
 		} else {
 			//otherwise we fetch posts from threads on different boards separarely
-//TODO: use big board:$or/postid:$in query so this can be tackled in a single db query
+			//TODO: use big board:$or/postid:$in query so this can be tackled in a single db query
 			await Promise.all(threads.map(async thread => {
 				//for each thread, fetch all posts from the matching board and thread matching the threads postId
 				const currentThreadPosts = await Posts.getThreadPosts(thread.board, thread.postId);
@@ -61,7 +64,7 @@ module.exports = async (posts, board, all=false) => {
 
 	if (postFiles.length > 0) {
 		const fileNames = postFiles.map(x => x.filename)//[...new Set(postFiles.map(x => x.filename))];
-        await Files.decrement(fileNames);
+		await Files.decrement(fileNames);
 		if (pruneImmediately) {
 			await pruneFiles(fileNames);
 		}
@@ -112,6 +115,10 @@ module.exports = async (posts, board, all=false) => {
 
 	//deleting before remarkup so quotes are accurate
 	const deletedPosts = await Posts.deleteMany(postMongoIds).then(result => result.deletedCount);
+	//emit the deletes to thread sockets (not recent sockets [yet?])
+	for (let i = 0; i < deleteEmits.length; i++) {
+		Socketio.emitRoom(deleteEmits[i].room, 'markPost', { postId: deleteEmits[i].postId, type: 'delete', mark: 'Deleted' });
+	}
 
 	if (all === false) {
 		//get posts that quoted deleted posts so we can remarkup them
@@ -125,18 +132,18 @@ module.exports = async (posts, board, all=false) => {
 					message = sanitize(quotedMessage, sanitizeOptions.after);
 					bulkWrites.push({
 						'updateOne': {
-		                	'filter': {
-		                    	'_id': post._id
-		                	},
-		                	'update': {
-		                    	'$set': {
-		                        	'quotes': threadQuotes,
+							'filter': {
+								'_id': post._id
+							},
+							'update': {
+								'$set': {
+									'quotes': threadQuotes,
 									'crossquotes': crossQuotes,
 									'message': message
-		                    	}
-		                	}
-		            	}
-		        	});
+								}
+							}
+						}
+					});
 				}
 			}));
 		}
@@ -147,6 +154,14 @@ module.exports = async (posts, board, all=false) => {
 		await Posts.db.bulkWrite(bulkWrites);
 	}
 
+	if (threads.length > 0) {
+		//delete the html/json for threads
+		await Promise.all(threads.map(thread => {
+			remove(`${uploadDirectory}/html/${thread.board}/thread/${thread.postId}.html`)
+			remove(`${uploadDirectory}/json/${thread.board}/thread/${thread.postId}.json`)
+		}));
+	}
+
 	//hooray!
 	return { action: deletedPosts > 0, message:`Deleted ${threads.length > 0 ? (threads.length + ' thread' + (threads.length > 1 ? 's' : '')) : ''} ${threads.length > 0 && deletedPosts-threads.length > 0 ? 'and' : ''} ${deletedPosts-threads.length > 0 ? (deletedPosts-threads.length + ' post' + (deletedPosts-threads.length > 1 ? 's' : '')) : ''}` };
 

models/forms/makepost.js

@@ -34,7 +34,7 @@ const path = require('path')
 module.exports = async (req, res, next) => {
 
 	const { checkRealMimeTypes, thumbSize, thumbExtension, videoThumbPercentage,
-		strictFiltering, animatedGifThumbnails, audioThumbnails } = config.get;
+		strictFiltering, animatedGifThumbnails, audioThumbnails, ipHashPermLevel } = config.get;
 
 	//spam/flood check
 	const flood = await spamCheck(req, res);
@@ -53,11 +53,12 @@ module.exports = async (req, res, next) => {
 	let thread = null;
 	const { filterBanDuration, filterMode, filters, blockedCountries, threadLimit, ids, userPostSpoiler,
 		lockReset, captchaReset, pphTrigger, tphTrigger, tphTriggerAction, pphTriggerAction,
-		maxFiles, sageOnlyEmail, forceAnon, replyLimit, disableReplySubject,
+		sageOnlyEmail, forceAnon, replyLimit, disableReplySubject,
 		captchaMode, lockMode, allowedFileTypes, customFlags, geoFlags, fileR9KMode, messageR9KMode } = res.locals.board.settings;
 	if (res.locals.permLevel >= 4
 		&& res.locals.country
 		&& blockedCountries.includes(res.locals.country.code)) {
+		await deleteTempFiles(req).catch(e => console.error);
 		return dynamicResponse(req, res, 403, 'message', {
 			'title': 'Forbidden',
 			'message': `Your country "${res.locals.country.name}" is not allowed to post on this board`,
@@ -614,9 +615,13 @@ ${res.locals.numFiles > 0 ? req.files.file.map(f => f.name+'|'+(f.phash || '')).
 	const { raw, single } = data.ip;
 	//but emit it to manage pages because they need to get all posts through socket including thread
 	Socketio.emitRoom('globalmanage-recent-hashed', 'newPost', { ...projectedPost, ip: { single: single.slice(-10), raw: null } });
-	Socketio.emitRoom('globalmanage-recent-raw', 'newPost', { ...projectedPost, ip: { single: single.slice(-10), raw } });
 	Socketio.emitRoom(`${res.locals.board._id}-manage-recent-hashed`, 'newPost', { ...projectedPost, ip: { single: single.slice(-10), raw: null } });
-	Socketio.emitRoom(`${res.locals.board._id}-manage-recent-raw`, 'newPost', { ...projectedPost, ip: { single: single.slice(-10), raw } });
+	if (ipHashPermLevel > -1) {
+		//small optimisation for boards where this is manually set to -1 for privacy, no need to emit to rooms that cant be accessed
+		//even if they are empty it will create extra communication noise in redis, socket adapter, etc.
+		Socketio.emitRoom('globalmanage-recent-raw', 'newPost', { ...projectedPost, ip: { single: single.slice(-10), raw } });
+		Socketio.emitRoom(`${res.locals.board._id}-manage-recent-raw`, 'newPost', { ...projectedPost, ip: { single: single.slice(-10), raw } });
+	}
 
 	//now add other pages to be built in background
 	if (enableCaptcha) {

models/forms/moveposts.js

@@ -3,6 +3,7 @@
 const uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js')
 	, { remove } = require('fs-extra')
 	, { Posts } = require(__dirname+'/../../db/')
+	, Socketio = require(__dirname+'/../../socketio.js')
 	, quoteHandler = require(__dirname+'/../../helpers/posting/quotes.js')
 	, { markdown } = require(__dirname+'/../../helpers/posting/markdown.js')
 	, { createHash } = require('crypto')
@@ -20,21 +21,18 @@ module.exports = async (req, res) => {
 		return acc;
 	}, { threads: [], postIds: [], postMongoIds: [] });
 
-//console.log(threads, postIds, postMongoIds)
+	//maybe should filter these? because it will include threads from which child posts are already fetched in the action handler, unlike the deleteposts model
+	const moveEmits = res.locals.posts.reduce((acc, post) => {
+		acc.push({
+			room: `${post.board}-${post.thread || post.postId}`,
+			postId: post.postId,
+		});
+		return acc;
+	}, []);
 
 	const backlinkRebuilds = new Set();
 	const bulkWrites = [];
 
-	if (threads.length > 0) {
-		//threads moved, so their html/json doesnt need to exist anymore
-		await Promise.all(threads.map(thread => {
-			return Promise.all([
-				remove(`${uploadDirectory}/html/${thread.board}/thread/${thread.postId}.html`),
-				remove(`${uploadDirectory}/json/${thread.board}/thread/${thread.postId}.json`)
-			]);
-		}));
-   	}
-
 	//remove backlinks from selected posts that link to unselected posts
 	bulkWrites.push({
 		'updateMany': {
@@ -95,6 +93,7 @@ module.exports = async (req, res) => {
 		acc.replyfiles += p.files.length;
 		return acc;
 	}, { replyposts: 0, replyfiles: 0 });
+
 	bulkWrites.push({
 		'updateOne': {
 			'filter': {
@@ -112,6 +111,11 @@ module.exports = async (req, res) => {
 
 	const movedPosts = await Posts.move(postMongoIds, req.body.move_to_thread).then(result => result.modifiedCount);
 
+	//emit markPost moves
+	for (let i = 0; i < moveEmits.length; i++) {
+		Socketio.emitRoom(moveEmits[i].room, 'markPost', { postId: moveEmits[i].postId, type: 'move', mark: 'Moved' });
+	}
+
 	//get posts that quoted moved posts so we can remarkup them
 	if (backlinkRebuilds.size > 0) {
 		const remarkupPosts = await Posts.globalGetPosts([...backlinkRebuilds]);
@@ -160,8 +164,6 @@ module.exports = async (req, res) => {
 		}));
 	}
 
-//console.log(require('util').inspect(bulkWrites, {depth:null}))
-
 /*
 - post A quotes B, then A is moved to another thread: WORKS (removes backlink on B)
 - moving post A back into thread with B and backlink gets readded: WORKS
@@ -174,6 +176,16 @@ module.exports = async (req, res) => {
 		await Posts.db.bulkWrite(bulkWrites);
 	}
 
+	//delete html/json for no longer existing threads, because op was moved
+	if (threads.length > 0) {
+		await Promise.all(threads.map(thread => {
+			return Promise.all([
+				remove(`${uploadDirectory}/html/${thread.board}/thread/${thread.postId}.html`),
+				remove(`${uploadDirectory}/json/${thread.board}/thread/${thread.postId}.json`)
+			]);
+		}));
+   	}
+
 	return {
 		message: 'Moved posts',
 		action: movedPosts > 0,

models/pages/csrf.js

@@ -0,0 +1,9 @@
+'use strict';
+
+module.exports = async (req, res, next) => {
+
+	res.json({
+		token: req.csrfToken(),
+	});
+
+}

models/pages/index.js

@@ -12,6 +12,7 @@ module.exports = {
 	catalog: require(__dirname+'/catalog.js'),
 	banners: require(__dirname+'/banners.js'),
 	customPage: require(__dirname+'/custompage.js'),
+	csrfPage: require(__dirname+'/csrf.js'),
 	randombanner: require(__dirname+'/randombanner.js'),
 	news: require(__dirname+'/news.js'),
 	captchaPage: require(__dirname+'/captchapage.js'),

package.json

@@ -1,10 +1,11 @@
 {
   "name": "jschan",
-  "version": "0.1.10",
-  "migrateVersion": "0.1.10",
+  "version": "0.2.0",
+  "migrateVersion": "0.2.0",
   "description": "",
   "main": "server.js",
   "dependencies": {
+    "@fatchan/express-fileupload": "^1.3.1",
     "bcrypt": "^5.0.1",
     "bull": "^3.29.3",
     "cache-pug-templates": "^2.0.3",
@@ -14,7 +15,6 @@
     "del": "^6.0.0",
     "dnsbl": "^3.2.0",
     "express": "^4.17.1",
-    "express-fileupload": "git+https://gitgud.io/fatchan/express-fileupload.git#ded0b1e69c222d1102fb2d8415996973e74a03fe",
     "express-session": "^1.17.2",
     "file-type": "^16.5.3",
     "fluent-ffmpeg": "^2.1.2",

views/custompages/faq.pug

@@ -11,7 +11,7 @@ block content
 		.anchor
 		table
 			tr
-				th Frequently Asked Questions 
+				th Frequently Asked Questions
 			tr
 				td.post-message
 					b General
@@ -21,7 +21,7 @@ block content
 						li: a(href='#contact') How can I contact the administration?
 					b Making posts
 					ul.mv-0
-						li: a(href='#captcha') How do I solve CAPTCHA?
+						li: a(href='#captcha') How do I solve the CAPTCHA?
 						li: a(href='#name-formatting') How do names, tripcodes and capcodes work?
 						li: a(href='#post-styling') What kind of styling options are available when making a post?
 						li: a(href='#post-info') What is the file size limit?
@@ -48,10 +48,10 @@ block content
 		.anchor#captcha
 		table
 			tr
-				th: a(href='#captcha') How do I solve CAPTCHA?
+				th: a(href='#captcha') How do I solve the CAPTCHA?
 			tr
 				td.post-message
-					p Select the boxes that correspond to each solid/filled icon in the grid. The image is shuffled and distorted, so use your brain.
+					| See the #[a(rel='nofollow' referrerpolicy='same-origin' target='_blank' href='http://fatchan.gitgud.site/jschan-docs/#captcha-block-bypass') API docs] for example captchas and solutions.
 	.table-container.flex-center.mv-5
 		.anchor#name-formatting
 		table
@@ -206,11 +206,12 @@ block content
 						| ( ・ω・) Let's try that again.
 			tr
 				td(colspan=2)
-					| The "language" of code blocks is optional. Without it, automatic language detection is used. 
-					| If the language is "plain", highlighting is disabled for the code block. If "aa" is used, the font will be adjusted for Japanese Shift JIS art. 
-					| Not all programming languages are supported, a subset of popular languages is used. 
-					| If the language is not in the supported list, the code block will be rendered like "plain" with no highlighting.
-					| Languages supported: #{codeLanguages.join(', ')}
+					| Supported languages for code block syntax highlighting: 
+					a(rel='nofollow' referrerpolicy='same-origin' target='_blank' href='https://github.com/highlightjs/highlight.js/blob/master/SUPPORTED_LANGUAGES.md') https://github.com/highlightjs/highlight.js/blob/master/SUPPORTED_LANGUAGES.md
+					| . 
+					| If you do not specify a language, a subset of languages is supported for auto-detection: #{codeLanguages.join(', ')}. 
+					| If the language is "plain", an unsupported value, or the auto-detect confidence is too low, highlighting is disabled for the code block. 
+					| If the language is "aa", the font will be adjusted for Japanese Shift JIS art.
 	.table-container.flex-center.mv-5
 		.anchor#post-info
 		table

views/includes/postform.pug

@@ -81,4 +81,4 @@ section.form-wrapper.flex-center
 			else
 				include ./captchaexpand.pug
 		input#submitpost(type='submit', value=`New ${isThread ? 'Reply' : 'Thread'}`)
-	a.collapse.no-decoration(href='#postform') [#{isThread ? 'New Reply' : 'New Thread'}]
+	a.collapse.no-decoration.post-button(href='#postform') [#{isThread ? 'New Reply' : 'New Thread'}]

views/mixins/modal.pug

@@ -67,6 +67,10 @@ mixin modal(data)
 								label.postform-style.ph-5
 									input#hidepoststubs-setting(type='checkbox')
 								.rlabel Hide post stubs
+							.row
+								label.postform-style.ph-5
+									input#hidedeletedpostcontent-setting(type='checkbox')
+								.rlabel Hide deleted post content
 							.row
 								label.postform-style.ph-5
 									input#disableboardcss-setting(type='checkbox')
@@ -166,9 +170,6 @@ mixin modal(data)
 								tbody#advancedfilters
 									tr
 										th(colspan=4) Post Filters
-										th
-										th
-										th
 									tr
 										td Type
 										td Value

views/pages/globalmanagesettings.pug

@@ -291,7 +291,7 @@ block content
 						h4.mv-5 Code Highlighting
 					.row
 						.label
-							| Code Highlighting Languages
+							| Code Highlighting Auto-Detect Languages
 							|
 							small
 								| (
@@ -415,6 +415,10 @@ block content
 						.label Hide Post Stubs
 						label.postform-style.ph-5
 							input(type='checkbox', name='frontend_script_default_hide_post_stubs', value='true' checked=settings.frontendScriptDefault.hidePostStubs)
+					.row
+						.label Hide Deleted Post Content
+						label.postform-style.ph-5
+							input(type='checkbox', name='frontend_script_default_hide_deleted_post_content', value='true' checked=settings.frontendScriptDefault.hideDeletedPostContent)
 					.row
 						.label Smooth Scrolling
 						label.postform-style.ph-5

views/pages/modlog.pug

@@ -29,7 +29,7 @@ block content
 						td
 							if log.showLinks
 								for postLink in log.postLinks
-									a(href=`/${board._id}/thread/${postLink.thread || postLink.postId}.html#${postLink.postId}`) >>#{postLink.postId}
+									a.quote(href=`/${board._id}/thread/${postLink.thread || postLink.postId}.html#${postLink.postId}`) >>#{postLink.postId}
 									|   
 							else
 								| #{log.postLinks.map(l => l.postId)}

views/pages/thread.pug

@@ -53,7 +53,7 @@ block content
 			for post in thread.replies
 				- uids && post.userId && uids.add(post.userId)
 				+post(post)
-		a.bottom-reply.no-decoration(href='#postform') [New Reply]
+		a.bottom-reply.no-decoration.post-button(href='#postform') [New Reply]
 		hr(size=1)
 		.wrapbar
 			if modview