From 24574862a2cbddad2c347ade9d772e4623277431 Mon Sep 17 00:00:00 2001 From: Thomas Lynch Date: Sat, 12 Sep 2020 06:09:58 +0000 Subject: [PATCH] Add file-type moodule to check file mime types strictly, with 2 optiosn in config about it Update express-fileupload dependency to clean tempfiles on numFilesLimitHandler Add a proper error message for max num files instead of allowing unlimited and limiting in board post method --- configs/main.js.example | 8 ++++- helpers/filemiddlewares.js | 12 +++++-- helpers/files/mimetypes.js | 12 +++++++ models/forms/makepost.js | 17 +++++++-- models/forms/uploadbanners.js | 15 ++++++++ models/pages/login.js | 4 +-- package-lock.json | 66 +++++++++++++++++++++++++++++++++-- package.json | 3 +- 8 files changed, 127 insertions(+), 10 deletions(-) diff --git a/configs/main.js.example b/configs/main.js.example index 54694d4a..bb274beb 100644 --- a/configs/main.js.example +++ b/configs/main.js.example @@ -124,9 +124,15 @@ module.exports = { "other files" section which includes an example configuration for .txt files to match this default config. mime types and file extention does not always correspond exactly this cant be done automatically. */ otherMimeTypes: [ - 'text/plain' + 'text/plain', + 'application/pdf' ], + //check the real mime type of uploaded files + checkRealMimeTypes: false, + //if checking real mime types, and the real type is unknown, allow it anyway + allowMimeNoMatch: false, + //default ban duration in ms if ban duration field is left blank (default value is 1 year) defaultBanDuration: 31536000000, diff --git a/helpers/filemiddlewares.js b/helpers/filemiddlewares.js index a75c99bc..636580ed 100644 --- a/helpers/filemiddlewares.js +++ b/helpers/filemiddlewares.js @@ -9,6 +9,13 @@ const { globalLimits, debugLogs, filterFileNames, spaceFileNameReplacement } = r 'redirect': req.headers.referer }); } + , numFilesUploadLimitFunction = (req, res, next) => { + return dynamicResponse(req, res, 400, 'message', { + 'title': 'Too many files', + 'message': 'You sent too many files in one request', + 'redirect': req.headers.referer + }); + } , upload = require('express-fileupload') , postFiles = upload({ debug: debugLogs, @@ -19,8 +26,9 @@ const { globalLimits, debugLogs, filterFileNames, spaceFileNameReplacement } = r limits: { totalSize: globalLimits.postFilesSize.max, fileSize: globalLimits.postFilesSize.max, - //files: globalLimits.postFiles.max + files: globalLimits.postFiles.max }, + numFilesLimitHandler: numFilesUploadLimitFunction, limitHandler: uploadLimitFunction, useTempFiles: true, tempFileDir: __dirname+'/../tmp/' @@ -29,7 +37,7 @@ const { globalLimits, debugLogs, filterFileNames, spaceFileNameReplacement } = r module.exports = { - handleBannerFiles: upload({ + handleBannerFiles: upload({ debug: debugLogs, createParentPath: true, safeFileNames: filterFileNames, diff --git a/helpers/files/mimetypes.js b/helpers/files/mimetypes.js index 83a9c8d8..2d0b9a57 100644 --- a/helpers/files/mimetypes.js +++ b/helpers/files/mimetypes.js @@ -1,5 +1,8 @@ 'use strict'; +const FileType = require('file-type') + , { allowMimeNoMatch } = require(__dirname+'/../../configs/main.js'); + const image = new Set([ 'image/jpeg', 'image/pjpeg', @@ -41,6 +44,15 @@ module.exports = { (options.other && other.has(mimetype)); }, + realMimeCheck: async (file) => { + const supposedMimeType = file.mimetype; + const realMimeType = await FileType.fromFile(file.tempFilePath); + if (!realMimeType) { + return allowMimeNoMatch; + } + return supposedMimeType === realMimeType.mime; + }, + image, animatedImage, video, audio, other }; diff --git a/models/forms/makepost.js b/models/forms/makepost.js index 2734002b..a0e50520 100644 --- a/models/forms/makepost.js +++ b/models/forms/makepost.js @@ -24,7 +24,7 @@ const path = require('path') , timeUtils = require(__dirname+'/../../helpers/timeutils.js') , deletePosts = require(__dirname+'/deletepost.js') , spamCheck = require(__dirname+'/../../helpers/checks/spamcheck.js') - , { thumbSize, thumbExtension, postPasswordSecret, strictFiltering } = require(__dirname+'/../../configs/main.js') + , { checkRealMimeTypes, thumbSize, thumbExtension, postPasswordSecret, strictFiltering } = require(__dirname+'/../../configs/main.js') , buildQueue = require(__dirname+'/../../queue.js') , dynamicResponse = require(__dirname+'/../../helpers/dynamic.js') , { buildThread } = require(__dirname+'/../../helpers/tasks.js'); @@ -161,7 +161,7 @@ module.exports = async (req, res, next) => { let files = []; // if we got a file if (res.locals.numFiles > 0) { - // check all mime types befoer we try saving anything + // check all mime types before we try saving anything for (let i = 0; i < res.locals.numFiles; i++) { if (!mimeTypes.allowed(req.files.file[i].mimetype, allowedFileTypes)) { await deleteTempFiles(req).catch(e => console.error); @@ -172,6 +172,19 @@ module.exports = async (req, res, next) => { }); } } + // check for any mismatching supposed mimetypes from the actual file mimetype + if (checkRealMimeTypes) { + for (let i = 0; i < res.locals.numFiles; i++) { + if (!(await mimeTypes.realMimeCheck(req.files.file[i]))) { + deleteTempFiles(req).catch(e => console.error); + return dynamicResponse(req, res, 400, 'message', { + 'title': 'Bad request', + 'message': `Mime type mismatch for file "${req.files.file[i].name}"`, + 'redirect': redirect + }); + } + } + } // then upload, thumb, get metadata, etc. for (let i = 0; i < res.locals.numFiles; i++) { const file = req.files.file[i]; diff --git a/models/forms/uploadbanners.js b/models/forms/uploadbanners.js index 9cbc947c..8eeebd16 100644 --- a/models/forms/uploadbanners.js +++ b/models/forms/uploadbanners.js @@ -2,6 +2,7 @@ const path = require('path') , { remove, pathExists } = require('fs-extra') + , { checkRealMimeTypes } = require(__dirname+'/../../configs/main.js') , uploadDirectory = require(__dirname+'/../../helpers/files/uploadDirectory.js') , moveUpload = require(__dirname+'/../../helpers/files/moveupload.js') , mimeTypes = require(__dirname+'/../../helpers/files/mimetypes.js') @@ -34,6 +35,20 @@ module.exports = async (req, res, next) => { } } + // check for any mismatching supposed mimetypes from the actual file mimetype + if (checkRealMimeTypes) { + for (let i = 0; i < res.locals.numFiles; i++) { + if (!(await mimeTypes.realMimeCheck(req.files.file[i]))) { + deleteTempFiles(req).catch(e => console.error); + return dynamicResponse(req, res, 400, 'message', { + 'title': 'Bad request', + 'message': `Mime type mismatch for file "${req.files.file[i].name}"`, + 'redirect': redirect + }); + } + } + } + const filenames = []; for (let i = 0; i < res.locals.numFiles; i++) { const file = req.files.file[i]; diff --git a/models/pages/login.js b/models/pages/login.js index 9a4463cb..5b19c27d 100644 --- a/models/pages/login.js +++ b/models/pages/login.js @@ -5,8 +5,8 @@ const { buildLogin } = require(__dirname+'/../../helpers/tasks.js') module.exports = async (req, res, next) => { - res.render('login', { + res.render('login', { 'goto': (typeof req.query.goto === 'string' ? req.query.goto : null) - }); + }); } diff --git a/package-lock.json b/package-lock.json index 5e838bb3..7fb0ecd8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -312,6 +312,11 @@ } } }, + "@tokenizer/token": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.1.1.tgz", + "integrity": "sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w==" + }, "@types/babel-types": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.7.tgz", @@ -330,6 +335,11 @@ "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" }, + "@types/debug": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", + "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" + }, "@types/events": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz", @@ -2718,6 +2728,17 @@ "resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz", "integrity": "sha1-EOhdo4v+p/xZk0HClu4ddyZu5kA=" }, + "file-type": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-15.0.0.tgz", + "integrity": "sha512-l0JCuF5F7NIybCfa9G2H0lKhhGaf0z+HJyLOmB2feknY7/HBVNyD4PLesGKLGqznwyVXGNnfpIOr+Fvca6bOEg==", + "requires": { + "readable-web-to-node-stream": "^2.0.0", + "strtok3": "^6.0.3", + "token-types": "^2.0.0", + "typedarray-to-buffer": "^3.1.5" + } + }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -3810,6 +3831,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", @@ -4114,8 +4140,7 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "optional": true + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "is-unc-path": { "version": "1.0.0", @@ -5315,6 +5340,11 @@ "pinkie-promise": "^2.0.0" } }, + "peek-readable": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-3.1.0.tgz", + "integrity": "sha512-KGuODSTV6hcgdZvDrIDBUkN0utcAVj1LL7FfGbM0viKTtCHmtZcuEJ+lGqsp0fTFkGqesdtemV2yUSMeyy3ddA==" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -6016,6 +6046,11 @@ "util-deprecate": "^1.0.1" } }, + "readable-web-to-node-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-2.0.0.tgz", + "integrity": "sha512-+oZJurc4hXpaaqsN68GoZGQAQIA3qr09Or4fqEsargABnbe5Aau8hFn6ISVleT3cpY/0n/8drn7huyyEvTbghA==" + }, "readdirp": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", @@ -7009,6 +7044,16 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" }, + "strtok3": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.0.4.tgz", + "integrity": "sha512-rqWMKwsbN9APU47bQTMEYTPcwdpKDtmf1jVhHzNW2cL1WqAxaM9iBb9t5P2fj+RV2YsErUWgQzHD5JwV0uCTEQ==", + "requires": { + "@tokenizer/token": "^0.1.1", + "@types/debug": "^4.1.5", + "peek-readable": "^3.1.0" + } + }, "supports-color": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", @@ -7198,6 +7243,15 @@ "resolved": "https://registry.npmjs.org/token-stream/-/token-stream-1.0.0.tgz", "integrity": "sha1-zCAOqyYT9BZtJ/+a/HylbUnfbrQ=" }, + "token-types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-2.0.0.tgz", + "integrity": "sha512-WWvu8sGK8/ZmGusekZJJ5NM6rRVTTDO7/bahz4NGiSDb/XsmdYBn6a1N/bymUHuWYTWeuLUg98wUzvE4jPdCZw==", + "requires": { + "@tokenizer/token": "^0.1.0", + "ieee754": "^1.1.13" + } + }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -7280,6 +7334,14 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "requires": { + "is-typedarray": "^1.0.0" + } + }, "uglify-js": { "version": "2.8.29", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", diff --git a/package.json b/package.json index 825db087..62fd05b8 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ "del": "^5.1.0", "dnsbl": "^3.2.0", "express": "^4.17.1", - "express-fileupload": "git+https://gitgud.io/fatchan/express-fileupload.git#64cf8d7afc13428bb02a1d158fb2b522cb29f134", + "express-fileupload": "git+https://gitgud.io/fatchan/express-fileupload.git#f152dbd33e87c66bbc423561192c98153fb7e957", "express-session": "^1.17.0", + "file-type": "^15.0.0", "fluent-ffmpeg": "^2.1.2", "fs": "0.0.1-security", "fs-extra": "^9.0.0",