diff --git a/lib/post/filteractions.js b/lib/post/filteractions.js new file mode 100644 index 00000000..aa46742e --- /dev/null +++ b/lib/post/filteractions.js @@ -0,0 +1,50 @@ +'use strict'; + +const { Bans } = require(__dirname+'/../../db/') + , dynamicResponse = require(__dirname+'/../misc/dynamic.js'); + +//ehhh, kinda too many args +module.exports = async (req, res, hitGlobalFilter, boardFilterMode, globalFilterMode, + boardFilterBanDuration, globalFilterBanDuration, filterBanAppealable, redirect) => { + + //global filter mode takes prio + const useFilterMode = hitGlobalFilter ? globalFilterMode : boardFilterMode; + + if (useFilterMode === 1) { + return dynamicResponse(req, res, 400, 'message', { + 'title': 'Bad request', + 'message': 'Your post was blocked by a word filter', + 'redirect': redirect + }); + } else { + const useFilterBanDuration = hitGlobalFilter ? globalFilterBanDuration : boardFilterBanDuration; + const banBoard = hitGlobalFilter ? null : res.locals.board._id; + const banDate = new Date(); + const banExpiry = new Date(useFilterBanDuration + banDate.getTime()); + const ban = { + 'ip': { + 'cloak': res.locals.ip.cloak, + 'raw': res.locals.ip.raw, + }, + 'type': res.locals.anonymizer ? 1 : 0, + 'range': 0, + 'reason': `${hitGlobalFilter ? 'global ' :''}word filter auto ban`, + 'board': banBoard, + 'posts': null, + 'issuer': 'system', //todo: make a "system" property instead? + 'date': banDate, + 'expireAt': banExpiry, + 'allowAppeal': hitGlobalFilter ? filterBanAppealable : true, + 'showUser': true, + 'seen': false + }; + const insertedResult = await Bans.insertOne(ban); + //add and delete some props for the dynamic response + ban._id = insertedResult.insertedId; + ban.ip.raw = null; + return dynamicResponse(req, res, 403, 'ban', { + bans: [ban] + }); + } + +}; diff --git a/lib/post/getfilterstrings.js b/lib/post/getfilterstrings.js new file mode 100644 index 00000000..1e423377 --- /dev/null +++ b/lib/post/getfilterstrings.js @@ -0,0 +1,35 @@ +'use strict'; + +module.exports = (req, res, strict=false) => { + + //combines a bunch of parts of the post (name, subject, message, filenames+phashes) + const fileStrings = res.locals.numFiles === 0 ? '' : req.files.file.map(f => `${f.name}|${f.phash || ''}`).join('|'); + const combinedString = [req.body.name, req.body.message, req.body.subject, req.body.email, fileStrings].join('|').toLowerCase(); + let strictCombinedString = combinedString; + + //"strict" filtering adds a bunch of permutations to also compare filters with; + if (strict === true) { + + //diacritics and "zalgo" removed + strictCombinedString += combinedString.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); + + //zero width spaces removed + strictCombinedString += combinedString.replace(/[\u200B-\u200D\uFEFF]/g, ''); + + //just a-z, 0-9, . and - + strictCombinedString += combinedString.replace(/[^a-zA-Z0-9.-]+/gm, ''); + + //urlendoded characters in URLs replaced (todo: remove this if/when the url regex gets updated to no longer match these) + strictCombinedString += combinedString.split(/(\%[^\%]+)/).map(part => { + try { + return decodeURIComponent(part); + } catch(e) { + return ''; + } + }).join(''); + + } + + return [combinedString, strictCombinedString]; + +}; diff --git a/models/forms/editpost.js b/models/forms/editpost.js index d44556f5..50d6dd34 100644 --- a/models/forms/editpost.js +++ b/models/forms/editpost.js @@ -7,6 +7,8 @@ const { Posts, Bans, Modlogs } = require(__dirname+'/../../db/') , { prepareMarkdown } = require(__dirname+'/../../lib/post/markdown/markdown.js') , messageHandler = require(__dirname+'/../../lib/post/message.js') , nameHandler = require(__dirname+'/../../lib/post/name.js') + , getFilterStrings = require(__dirname+'/../../lib/post/getfilterstrings.js') + , filterActions = require(__dirname+'/../../lib/post/filteractions.js') , config = require(__dirname+'/../../lib/misc/config.js') , buildQueue = require(__dirname+'/../../lib/build/queue.js') , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js') @@ -27,54 +29,18 @@ todo: handle some more situations const { board, post } = res.locals; //filters - if (res.locals.permissions.get(Permissions.BYPASS_FILTERS)) { //global staff bypass filters for edit - const globalSettings = config.get; - if (globalSettings && globalSettings.filters.length > 0 && globalSettings.filterMode > 0) { + if (res.locals.permissions.get(Permissions.BYPASS_FILTERS)) { + //only global filters are checked, because anybody who could edit bypasses board filters + const { filters, filterMode, filterBanDuration } = config.get; + if (filters.length > 0 && filterMode > 0) { let hitGlobalFilter = false - , ban - , concatContents = `|${req.body.name}|${req.body.message}|${req.body.subject}|${req.body.email}|${res.locals.numFiles > 0 ? req.files.file.map(f => f.name).join('|') : ''}`.toLowerCase() - , allContents = concatContents; - if (strictFiltering) { - allContents += concatContents.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); //removing diacritics - allContents += concatContents.replace(/[\u200B-\u200D\uFEFF]/g, ''); //removing ZWS - allContents += concatContents.replace(/[^a-zA-Z0-9.-]+/gm, ''); //removing anything thats not alphamnumeric or . and - - allContents += concatContents.split(/(\%[^\%]+)/).map(part => { try { return decodeURIComponent(part) } catch(e) { return '' } }).join(''); //catch pedophile spammers url-fu with encoding - } - //global filters - hitGlobalFilter = globalSettings.filters.some(filter => { return allContents.includes(filter.toLowerCase()) }); + , ban; + const [combinedString, strictCombinedString] = getFilterStrings(req, res, strictFiltering); + hitGlobalFilter = filters.some(filter => { return allContents.includes(filter.toLowerCase()) }); + //block/ban edit if (hitGlobalFilter) { - if (globalSettings.filterMode === 1) { - return dynamicResponse(req, res, 400, 'message', { - 'title': 'Bad request', - 'message': 'Your edit was blocked by a global word filter', - }); - } else { - const banDate = new Date(); - const banExpiry = new Date(globalSettings.filterBanDuration + banDate.getTime()); - const ban = { - 'ip': { - 'cloak': res.locals.ip.cloak, - 'raw': res.locals.ip.raw, - }, - 'type': res.locals.anonymizer ? 1 : 0, - 'range': 0, - 'reason': 'global word filter auto ban', - 'board': null, - 'posts': null, - 'issuer': 'system', //what should i call this - 'date': banDate, - 'expireAt': banExpiry, - 'allowAppeal': filterBanAppealable, - 'showUser': true, - 'seen': false - }; - const insertedResult = await Bans.insertOne(ban); - ban._id = insertedResult.insertedId; - ban.ip.raw = null; //for dynamicresponse - return dynamicResponse(req, res, 403, 'ban', { - bans: [ban] - }); - } + return filterActions(req, res, hitGlobalFilter, 0, globalFilterMode, + 0, globalFilterBanDuration, null, filterBanAppealable, null); } } } diff --git a/models/forms/makepost.js b/models/forms/makepost.js index f7eb1251..6a464106 100644 --- a/models/forms/makepost.js +++ b/models/forms/makepost.js @@ -7,9 +7,11 @@ const path = require('path') , uploadDirectory = require(__dirname+'/../../lib/file/uploaddirectory.js') , Mongo = require(__dirname+'/../../db/db.js') , Socketio = require(__dirname+'/../../lib/misc/socketio.js') - , { Stats, Posts, Boards, Files, Bans } = require(__dirname+'/../../db/') + , { Stats, Posts, Boards, Files } = require(__dirname+'/../../db/') , cache = require(__dirname+'/../../lib/redis/redis.js') , nameHandler = require(__dirname+'/../../lib/post/name.js') + , getFilterStrings = require(__dirname+'/../../lib/post/getfilterstrings.js') + , filterActions = require(__dirname+'/../../lib/post/filteractions.js') , { prepareMarkdown } = require(__dirname+'/../../lib/post/markdown/markdown.js') , messageHandler = require(__dirname+'/../../lib/post/message.js') , moveUpload = require(__dirname+'/../../lib/file/moveupload.js') @@ -105,71 +107,39 @@ module.exports = async (req, res, next) => { }); } } + //filters if (!res.locals.permissions.get(Permissions.BYPASS_FILTERS)) { + + //deconstruct global filter settings to differnt names, else they would conflict with the respective board-level setting const { filters: globalFilters, filterMode: globalFilterMode, filterBanDuration: globalFilterBanDuration } = config.get; + let hitGlobalFilter = false , hitLocalFilter = false , ban; - let concatContents = `|${req.body.name}|${req.body.message}|${req.body.subject}|${req.body.email}|\ -${res.locals.numFiles > 0 ? req.files.file.map(f => f.name+'|'+(f.phash || '')).join('|') : ''}`.toLowerCase(); - let allContents = concatContents; - if (strictFiltering || res.locals.board.settings.strictFiltering) { //strict filtering adds a few transformations of the text to try and match filters when sers use techniques like zalgo, ZWS, markdown, multi-line, etc. - allContents += concatContents.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); //removing diacritics - allContents += concatContents.replace(/[\u200B-\u200D\uFEFF]/g, ''); //removing ZWS - allContents += concatContents.replace(/[^a-zA-Z0-9.-]+/gm, ''); //removing anything thats not alphamnumeric or . and - - allContents += concatContents.split(/(\%[^\%]+)/).map(part => { try { return decodeURIComponent(part) } catch(e) { return '' } }).join(''); //catch pedophile spammers url-fu with encoding - } - //global filters + let [combinedString, strictCombinedString] = getFilterStrings(req, res, strictFiltering || res.locals.board.settings.strictFiltering); + + //compare to global filters if (globalFilters && globalFilters.length > 0 && globalFilterMode > 0) { - hitGlobalFilter = globalFilters.some(filter => { return allContents.includes(filter.toLowerCase()) }); + hitGlobalFilter = globalFilters.some(filter => { return strictCombinedString.includes(filter.toLowerCase()) }); } - //board-specific filters + + //compare to board filters if (!hitGlobalFilter && !res.locals.permissions.get(Permissions.MANAGE_BOARD_GENERAL) && filterMode > 0 && filters && filters.length > 0) { - const localFilterContents = res.locals.board.settings.strictFiltering ? allContents : concatContents; + const localFilterContents = res.locals.board.settings.strictFiltering === true ? strictCombinedString : combinedString; hitLocalFilter = filters.some(filter => { return localFilterContents.includes(filter.toLowerCase()) }); } + + //block post/apply bans if an active filter matched if (hitGlobalFilter || hitLocalFilter) { await deleteTempFiles(req).catch(e => console.error); - const useFilterMode = hitGlobalFilter ? globalFilterMode : filterMode; //global override local filter - if (useFilterMode === 1) { - return dynamicResponse(req, res, 400, 'message', { - 'title': 'Bad request', - 'message': 'Your post was blocked by a word filter', - 'redirect': redirect - }); - } else { //otherwise filter mode must be 2 - const useFilterBanDuration = hitGlobalFilter ? globalFilterBanDuration : filterBanDuration; - const banBoard = hitGlobalFilter ? null : res.locals.board._id; - const banDate = new Date(); - const banExpiry = new Date(useFilterBanDuration + banDate.getTime()); - const ban = { - 'ip': { - 'cloak': res.locals.ip.cloak, - 'raw': res.locals.ip.raw, - }, - 'type': res.locals.anonymizer ? 1 : 0, - 'range': 0, - 'reason': `${hitGlobalFilter ? 'global ' :''}word filter auto ban`, - 'board': banBoard, - 'posts': null, - 'issuer': 'system', //what should i call this - 'date': banDate, - 'expireAt': banExpiry, - 'allowAppeal': hitGlobalFilter ? filterBanAppealable : true, - 'showUser': true, - 'seen': false - }; - const insertedResult = await Bans.insertOne(ban); - ban._id = insertedResult.insertedId; - ban.ip.raw = null; //for dynamicresponse - return dynamicResponse(req, res, 403, 'ban', { - bans: [ban] - }); - } + return filterActions(req, res, hitGlobalFilter, filterMode, globalFilterMode, + filterBanDuration, globalFilterBanDuration, globalFilterBanDuration, + filterBanAppealable, redirect); } + } //for r9k messages. usually i wouldnt process these if its not enabled e.g. flags and IDs but in this case I think its necessary