Merge branch 'develop' into feature/396-localisation

CHANGELOG.md

@@ -22,6 +22,10 @@ Now, back to the program. Here are the changes for 1.0.0, with one especially no
   - Remove showing language and relevance data when auto detecting highlighted code block language
   - More minor bugfixes to permissions pages displays.
 
+### 0.11.4
+  - Bugfix for the message stating how many banners were deleted when deleting banners.
+  - Add an option to limit the total resolution of an image/video (width*height).
+
 ### 0.11.3
   - Fix max vs total upload count in controller for uploading board assets, flags and banners.
   - Move css theme assets to themes/assets instead of all lumped in one folder.

configs/nginx/snippets/jschan_clearnet_routes.conf

@@ -1,5 +1,6 @@
 location / {
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 
@@ -16,6 +17,7 @@ location / {
 
 location @backend {
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 	proxy_set_header X-Forwarded-Proto https;
@@ -29,6 +31,7 @@ location @backend {
 location @backend-private {
 	include /etc/nginx/snippets/security_headers_nocache.conf;
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 	proxy_set_header X-Forwarded-Proto https;

configs/nginx/snippets/jschan_loki_routes.conf

@@ -1,5 +1,6 @@
 location / {
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 
@@ -16,6 +17,7 @@ location / {
 
 location @backend {
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 	proxy_set_header X-Forwarded-Proto http;
@@ -28,6 +30,8 @@ location @backend {
 
 location @backend-private {
 	include /etc/nginx/snippets/security_headers_nocache.conf;
+	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 	proxy_set_header X-Forwarded-Proto http;

configs/nginx/snippets/jschan_tor_routes.conf

@@ -1,13 +1,12 @@
 location / {
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
-
 	proxy_set_header Host $host;
 	proxy_set_header Upgrade $http_upgrade;
 	proxy_set_header Connection 'upgrade';
 	proxy_cache_bypass $http_upgrade;
-
 	proxy_set_header X-Forwarded-Proto http;
 	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 	proxy_set_header X-Real-IP $remote_addr;
@@ -16,6 +15,7 @@ location / {
 
 location @backend {
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 	proxy_set_header X-Forwarded-Proto http;
@@ -29,6 +29,7 @@ location @backend {
 location @backend-private {
 	include /etc/nginx/snippets/security_headers_nocache.conf;
 	proxy_buffering off;
+	proxy_request_buffering off;
 	proxy_pass http://chan$request_uri;
 	proxy_http_version 1.1;
 	proxy_set_header X-Forwarded-Proto http;

configs/template.js.example

@@ -140,9 +140,9 @@ module.exports = {
 	//enable the webring (also copy configs/webring.json.example -> configs/webring.json and edit)
 	enableWebring: false,
 
-	//extension for thumbnails that do not contain transparency (png will be used)
-	thumbExtension: '.jpg',
-	//.gif images > thumbnail size will have animated .gif thumbnails, overriding thumbExtension
+	//extension for thumbnails
+	thumbExtension: '.webp',
+	//whether to animate gif thumbnails
 	animatedGifThumbnails: false,
 	//generate waveform thumbnails for audio
 	audioThumbnails: true,
@@ -264,10 +264,14 @@ module.exports = {
 			max: 1000
 		},
 		postFiles: { //number of files in a post
-			max: 5
+			max: 5,
 		},
-		postFilesSize: { //in bytes, 10MB default
-			max: 10485760
+		postFilesSize: {
+			max: 10485760, //in bytes, 10MB default
+			//width*height max number, 10000*10000 default, since square vs very wids vs very tall doesnt matter,
+			//just the raw number of pixels e.g. 4K 3840x2160=77414400, so we give PLENTY of headroom
+			imageResolution: 100000000,
+			videoResolution: 77414400,
 		},
 		bannerFiles: {
 			width: 300, //banner image max width in px

controllers/forms/globalsettings.js

@@ -34,7 +34,7 @@ module.exports = {
 			'board_defaults_pph_trigger', 'board_defaults_tph_trigger_action', 'board_defaults_pph_trigger_action', 'board_defaults_captcha_reset', 'board_defaults_lock_reset',
 			'board_defaults_thread_limit', 'board_defaults_reply_limit', 'board_defaults_bump_limit', 'board_defaults_max_files', 'board_defaults_min_thread_message_length',
 			'board_defaults_min_reply_message_length', 'board_defaults_max_thread_message_length', 'board_defaults_max_reply_message_length', 'board_defaults_filter_mode',
-			'board_defaults_delete_protection_count', 'frontend_script_default_tegaki_height', 'frontend_script_default_tegaki_width']
+			'board_defaults_delete_protection_count', 'frontend_script_default_tegaki_height', 'frontend_script_default_tegaki_width', 'global_limits_post_files_size_image_resolution', 'global_limits_post_files_size_video_resolution']
 	}),
 
 	controller: async (req, res, next) => {
@@ -142,6 +142,8 @@ module.exports = {
 			{ result: minmaxBody(req.body.global_limits_bump_limit_min, req.body.global_limits_bump_limit_max), expected: true, error: __('Global bump limit min must be less than max') },
 			{ result: numberBody(req.body.global_limits_post_files_max), expected: true, error: __('Post files max must be a number') },
 			{ result: numberBody(req.body.global_limits_post_files_size_max), expected: true, error: __('Post files size must be a number') },
+			{ result: numberBody(req.body.global_limits_post_files_size_image_resolution), expected: true, error: __('Image resolution max must be a number') },
+			{ result: numberBody(req.body.global_limits_post_files_size_video_resolution), expected: true, error: __('Video resolution max must be a number') },
 			{ result: numberBody(req.body.global_limits_banner_files_width, 1), expected: true, error: __('Banner files height must be a number > 0') },
 			{ result: numberBody(req.body.global_limits_banner_files_height, 1), expected: true, error: __('Banner files width must be a number > 0') },
 			{ result: numberBody(req.body.global_limits_banner_files_size_max), expected: true, error: __('Banner files size must be a number') },

lib/file/image/getdimensions.js

@@ -6,7 +6,7 @@ module.exports = (filename, folder, temp) => {
 	return new Promise((resolve, reject) => {
 		const filePath = temp === true ? filename : `${uploadDirectory}/${folder}/${filename}`;
 		gm(`${filePath}[0]`) //0 for first frame of gifs, much faster
-			.identify(function (err, data) {
+			.size(function (err, data) {
 				if (err) {
 					return reject(err);
 				}

migrations/0.11.4.js

@@ -0,0 +1,13 @@
+'use strict';
+
+module.exports = async(db, redis) => {
+	console.log('adding image/video resolution maximum setting');
+	await db.collection('globalsettings').updateOne({ _id: 'globalsettings' }, {
+		'$set': {
+			'globalLimits.postFilesSize.imageResolution': 100000000,
+			'globalLimits.postFilesSize.videoResolution': 77414400,
+		},
+	});
+	console.log('Clearing globalsettings cache');
+	await redis.deletePattern('globalsettings');
+};

models/forms/changeglobalsettings.js

@@ -221,6 +221,8 @@ module.exports = async (req, res) => {
 			},
 			postFilesSize: {
 				max: numberSetting(req.body.global_limits_post_files_size_max, oldSettings.globalLimits.postFilesSize.max),
+				imageResolution: numberSetting(req.body.global_limits_post_files_size_image_resolution, oldSettings.globalLimits.postFilesSize.imageResolution),
+				videoResolution: numberSetting(req.body.global_limits_post_files_size_video_resolution, oldSettings.globalLimits.postFilesSize.videoResolution),
 			},
 			bannerFiles: {
 				width: numberSetting(req.body.global_limits_banner_files_width, oldSettings.globalLimits.bannerFiles.width),

models/forms/deletebanners.js

@@ -13,14 +13,15 @@ module.exports = async (req, res) => {
 	const redirect = `/${req.params.board}/manage/assets.html`;
 
 	//delete file of all selected banners
-	await Promise.all(req.body.checkedbanners.map(async filename => {
+	await Promise.all(req.body.checkedbanners.map(filename => {
 		remove(`${uploadDirectory}/banner/${req.params.board}/${filename}`);
 	}));
 
 	//remove from db
-	const amount = await Boards.removeBanners(req.params.board, req.body.checkedbanners).then(result => result.modifiedCount);
+	await Boards.removeBanners(req.params.board, req.body.checkedbanners);
 
 	//update res locals banners in memory
+	const beforeBanners = res.locals.board.banners.length;
 	res.locals.board.banners = res.locals.board.banners.filter(banner => {
 		return !req.body.checkedbanners.includes(banner);
 	});
@@ -35,7 +36,7 @@ module.exports = async (req, res) => {
 
 	return dynamicResponse(req, res, 200, 'message', {
 		'title': __('Success'),
-		'message': __('Deleted %s banners', amount),
+		'message': __('Deleted %s banners', beforeBanners - res.locals.board.banners.length),
 		'redirect': redirect
 	});
 };

models/forms/makepost.js

@@ -16,7 +16,7 @@ const { createHash, randomBytes } = require('crypto')
 	, moveUpload = require(__dirname+'/../../lib/file/moveupload.js')
 	, mimeTypes = require(__dirname+'/../../lib/file/mimetypes.js')
 	, imageThumbnail = require(__dirname+'/../../lib/file/image/imagethumbnail.js')
-	, imageIdentify = require(__dirname+'/../../lib/file/image/imageidentify.js')
+	, getDimensions = require(__dirname+'/../../lib/file/image/getdimensions.js')
 	, videoThumbnail = require(__dirname+'/../../lib/file/video/videothumbnail.js')
 	, audioThumbnail = require(__dirname+'/../../lib/file/audio/audiothumbnail.js')
 	, ffprobe = require(__dirname+'/../../lib/file/ffprobe.js')
@@ -38,7 +38,7 @@ module.exports = async (req, res) => {
 
 	const { __ } = res.locals;
 	const { filterBanAppealable, checkRealMimeTypes, thumbSize, thumbExtension, videoThumbPercentage,
-		strictFiltering, animatedGifThumbnails, audioThumbnails, dontStoreRawIps } = config.get;
+		strictFiltering, animatedGifThumbnails, audioThumbnails, dontStoreRawIps, globalLimits } = config.get;
 
 	//spam/flood check
 	const flood = await spamCheck(req, res);
@@ -258,10 +258,9 @@ module.exports = async (req, res) => {
 				switch (type) {
 					case 'image': {
 						processedFile.thumbextension = thumbExtension;
-						///detect images with opacity for PNG thumbnails, set thumbextension before increment
-						let imageData;
+						let imageDimensions;
 						try {
-							imageData = await imageIdentify(req.files.file[i].tempFilePath, null, true);
+							imageDimensions = await getDimensions(req.files.file[i].tempFilePath, null, true);
 						} catch (e) {
 							await deleteTempFiles(req).catch(console.error);
 							return dynamicResponse(req, res, 400, 'message', {
@@ -270,15 +269,20 @@ module.exports = async (req, res) => {
 								'redirect': redirect
 							});
 						}
-						if (imageData['Channel Statistics'] && imageData['Channel Statistics']['Opacity']) {
-							//does this depend on GM version or anything?
-							const opacityMaximum = imageData['Channel Statistics']['Opacity']['Maximum'];
-							if (opacityMaximum !== '0.00 (0.0000)') {
-								processedFile.thumbextension = '.png';
-							}
+						if (Math.floor(imageDimensions.width*imageDimensions.height) > globalLimits.postFilesSize.imageResolution) {
+							await deleteTempFiles(req).catch(console.error);
+							return dynamicResponse(req, res, 400, 'message', {
+								'title': 'Bad request',
+								'message': `File "${req.files.file[i].name}" image resolution is too high. Width*Height must not exceed ${globalLimits.postFilesSize.imageResolution}.`,
+								'redirect': redirect
+							});
+						}
+						if (thumbExtension === '.jpg' && subtype === 'png') {
+							//avoid transparency issues for jpg thumbnails on pngs (the most common case -- for anything else, use webp thumbExtension)
+							processedFile.thumbextension = '.png';
 						}
-						processedFile.geometry = imageData.size;
-						processedFile.geometryString = imageData.Geometry;
+						processedFile.geometry = imageDimensions;
+						processedFile.geometryString = `${imageDimensions.width}x${imageDimensions.height}`;
 						const lteThumbSize = (processedFile.geometry.height <= thumbSize
 							&& processedFile.geometry.width <= thumbSize);
 						processedFile.hasThumb = !(mimeTypes.allowed(file.mimetype, {image: true})
@@ -287,8 +291,6 @@ module.exports = async (req, res) => {
 						let firstFrameOnly = true;
 						if (processedFile.hasThumb //if it needs thumbnailing
 							&& (file.mimetype === 'image/gif' //and its a gif
-//								&& !lteThumbSize //and its big enough -> why was this a thing originally?
-								&& (imageData['Delay'] != null || imageData['Iterations'] != null) //and its not a static gif (naive check)
 								&& animatedGifThumbnails === true)) { //and animated thumbnails for gifs are enabled
 							firstFrameOnly = false;
 							processedFile.thumbextension = '.gif';
@@ -310,6 +312,14 @@ module.exports = async (req, res) => {
 						if (videoStreams.length > 0) {
 							processedFile.thumbextension = thumbExtension;
 							processedFile.geometry = {width: videoStreams[0].coded_width, height: videoStreams[0].coded_height};
+							if (Math.floor(processedFile.geometry.width*processedFile.geometry.height) > globalLimits.postFilesSize.videoResolution) {
+								await deleteTempFiles(req).catch(console.error);
+								return dynamicResponse(req, res, 400, 'message', {
+									'title': 'Bad request',
+									'message': `File "${req.files.file[i].name}" video resolution is too high. Width*Height must not exceed ${globalLimits.postFilesSize.videoResolution}.`,
+									'redirect': redirect
+								});
+							}
 							processedFile.geometryString = `${processedFile.geometry.width}x${processedFile.geometry.height}`;
 							processedFile.hasThumb = true;
 							await saveFull();

models/forms/uploadbanners.js

@@ -5,7 +5,7 @@ const { remove, pathExists } = require('fs-extra')
 	, uploadDirectory = require(__dirname+'/../../lib/file/uploaddirectory.js')
 	, moveUpload = require(__dirname+'/../../lib/file/moveupload.js')
 	, mimeTypes = require(__dirname+'/../../lib/file/mimetypes.js')
-	, imageIdentify = require(__dirname+'/../../lib/file/image/imageidentify.js')
+	, getDimensions = require(__dirname+'/../../lib/file/image/getdimensions.js')
 	, deleteTempFiles = require(__dirname+'/../../lib/file/deletetempfiles.js')
 	, dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js')
 	, { Boards } = require(__dirname+'/../../db/')
@@ -47,8 +47,8 @@ module.exports = async (req, res) => {
 		}
 
 		//300x100 check
-		const imageData = await imageIdentify(req.files.file[i].tempFilePath, null, true);
-		let geometry = imageData.size;
+		const imageDimensions = await getDimensions(req.files.file[i].tempFilePath, null, true);
+		let geometry = imageDimensions;
 		if (Array.isArray(geometry)) {
 			geometry = geometry[0];
 		}

package.json

@@ -1,7 +1,7 @@
 {
   "name": "jschan",
-  "version": "0.11.3",
-  "migrateVersion": "0.11.0",
+  "version": "0.11.4",
+  "migrateVersion": "0.11.4",
   "description": "",
   "main": "server.js",
   "dependencies": {
@@ -39,8 +39,8 @@
     "imghash": "^0.0.9",
     "ioredis": "^4.28.5",
     "ip6addr": "^0.2.5",
-    "mongodb": "^4.13.0",
-    "node-fetch": "^2.6.7",
+    "mongodb": "^4.14.0",
+    "node-fetch": "^2.6.9",
     "otpauth": "^9.0.2",
     "path": "^0.12.7",
     "pm2": "^5.2.2",
@@ -48,16 +48,16 @@
     "pug-runtime": "^3.0.1",
     "qrcode": "^1.5.1",
     "redlock": "^4.2.0",
-    "sanitize-html": "^2.8.1",
+    "sanitize-html": "^2.10.0",
     "saslprep": "^1.0.3",
     "semver": "^7.3.8",
-    "socket.io": "^4.5.4",
+    "socket.io": "^4.6.1",
     "socks-proxy-agent": "^6.2.1",
     "uid-safe": "^2.1.5",
     "unix-crypt-td-js": "^1.1.4"
   },
   "devDependencies": {
-    "eslint": "^8.30.0",
+    "eslint": "^8.36.0",
     "eslint-plugin-jest": "^26.9.0",
     "jest": "^27.5.1",
     "jest-cli": "^27.5.1",

views/pages/globalmanagesettings.pug

@@ -94,9 +94,6 @@ block content
 																			.label #{__('Prune Files Immediately')}
 																			label.postform-style.ph-5
 																				input(type='checkbox', name='prune_immediately', value='true' checked=settings.pruneImmediately)
-																		.row
-																			.label #{__('Thumbnail File Extension')}
-																			input(type='text' name='thumb_extension' value=settings.thumbExtension)
 																		.row
 																			.label #{__('Fuzzy Hash Images')}
 																			label.postform-style.ph-5
@@ -579,6 +576,15 @@ block content
 																		.row
 																			.label #{__('Space File Name Replacement')}
 																			input(type='text', name='space_file_name_replacement', value=settings.spaceFileNameReplacement)
+																		.row
+																			.label #{__('Thumbnail File Extension')}
+																			input(type='text' name='thumb_extension' value=settings.thumbExtension)
+																		.row
+																			.label #{__('Image Max Resolution')}
+																			input(type='text' name='global_limits_post_files_size_image_resolution' value=settings.globalLimits.postFilesSize.imageResolution)
+																		.row
+																			.label #{__('Video Max Resolution')}
+																			input(type='text' name='global_limits_post_files_size_video_resolution' value=settings.globalLimits.postFilesSize.videoResolution)
 
 																.tab.tab-9
 																	.col