fatchan
/
jschan
mirror of https://gitgud.io/fatchan/jschan.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor, dedup the combining of post data into strings for filtering, and blocking post/applying ban. also improve the comments. previously was ugly and duplicated between makepost and editpost model

Browse Source
indiachan-spamvector
Thomas Lynch 2 years ago
parent f75fd6dfad
commit 7893947ee6
4 changed files with 117 additions and 96 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 50
      lib/post/filteractions.js
  2. 35
      lib/post/getfilterstrings.js
  3. 58
      models/forms/editpost.js
  4. 70
      models/forms/makepost.js

50
lib/post/filteractions.js
Unescape View File

@ -0,0 +1,50 @@
'use strict';
const { Bans } = require(__dirname+'/../../db/')
, dynamicResponse = require(__dirname+'/../misc/dynamic.js');
//ehhh, kinda too many args
module.exports = async (req, res, hitGlobalFilter, boardFilterMode, globalFilterMode,
boardFilterBanDuration, globalFilterBanDuration, filterBanAppealable, redirect) => {
//global filter mode takes prio
const useFilterMode = hitGlobalFilter ? globalFilterMode : boardFilterMode;
if (useFilterMode === 1) {
return dynamicResponse(req, res, 400, 'message', {
'title': 'Bad request',
'message': 'Your post was blocked by a word filter',
'redirect': redirect
});
} else {
const useFilterBanDuration = hitGlobalFilter ? globalFilterBanDuration : boardFilterBanDuration;
const banBoard = hitGlobalFilter ? null : res.locals.board._id;
const banDate = new Date();
const banExpiry = new Date(useFilterBanDuration + banDate.getTime());
const ban = {
'ip': {
'cloak': res.locals.ip.cloak,
'raw': res.locals.ip.raw,
},
'type': res.locals.anonymizer ? 1 : 0,
'range': 0,
'reason': `${hitGlobalFilter ? 'global ' :''}word filter auto ban`,
'board': banBoard,
'posts': null,
'issuer': 'system', //todo: make a "system" property instead?
'date': banDate,
'expireAt': banExpiry,
'allowAppeal': hitGlobalFilter ? filterBanAppealable : true,
'showUser': true,
'seen': false
};
const insertedResult = await Bans.insertOne(ban);
//add and delete some props for the dynamic response
ban._id = insertedResult.insertedId;
ban.ip.raw = null;
return dynamicResponse(req, res, 403, 'ban', {
bans: [ban]
});
}
};

35
lib/post/getfilterstrings.js
Unescape View File

@ -0,0 +1,35 @@
'use strict';
module.exports = (req, res, strict=false) => {
//combines a bunch of parts of the post (name, subject, message, filenames+phashes)
const fileStrings = res.locals.numFiles === 0 ? '' : req.files.file.map(f => `${f.name}|${f.phash || ''}`).join('|');
const combinedString = [req.body.name, req.body.message, req.body.subject, req.body.email, fileStrings].join('|').toLowerCase();
let strictCombinedString = combinedString;
//"strict" filtering adds a bunch of permutations to also compare filters with;
if (strict === true) {
//diacritics and "zalgo" removed
strictCombinedString += combinedString.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
//zero width spaces removed
strictCombinedString += combinedString.replace(/[\u200B-\u200D\uFEFF]/g, '');
//just a-z, 0-9, . and -
strictCombinedString += combinedString.replace(/[^a-zA-Z0-9.-]+/gm, '');
//urlendoded characters in URLs replaced (todo: remove this if/when the url regex gets updated to no longer match these)
strictCombinedString += combinedString.split(/(\%[^\%]+)/).map(part => {
try {
return decodeURIComponent(part);
} catch(e) {
return '';
}
}).join('');
}
return [combinedString, strictCombinedString];
};

58
models/forms/editpost.js
Unescape View File

@ -7,6 +7,8 @@ const { Posts, Bans, Modlogs } = require(__dirname+'/../../db/')
, { prepareMarkdown } = require(__dirname+'/../../lib/post/markdown/markdown.js')
, messageHandler = require(__dirname+'/../../lib/post/message.js')
, nameHandler = require(__dirname+'/../../lib/post/name.js')
, getFilterStrings = require(__dirname+'/../../lib/post/getfilterstrings.js')
, filterActions = require(__dirname+'/../../lib/post/filteractions.js')
, config = require(__dirname+'/../../lib/misc/config.js')
, buildQueue = require(__dirname+'/../../lib/build/queue.js')
, dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js')
@ -27,54 +29,18 @@ todo: handle some more situations
const { board, post } = res.locals;
//filters
if (res.locals.permissions.get(Permissions.BYPASS_FILTERS)) { //global staff bypass filters for edit
const globalSettings = config.get;
if (globalSettings && globalSettings.filters.length > 0 && globalSettings.filterMode > 0) {
if (res.locals.permissions.get(Permissions.BYPASS_FILTERS)) {
//only global filters are checked, because anybody who could edit bypasses board filters
const { filters, filterMode, filterBanDuration } = config.get;
if (filters.length > 0 && filterMode > 0) {
let hitGlobalFilter = false
, ban
, concatContents = `|${req.body.name}|${req.body.message}|${req.body.subject}|${req.body.email}|${res.locals.numFiles > 0 ? req.files.file.map(f => f.name).join('|') : ''}`.toLowerCase()
, allContents = concatContents;
if (strictFiltering) {
allContents += concatContents.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); //removing diacritics
allContents += concatContents.replace(/[\u200B-\u200D\uFEFF]/g, ''); //removing ZWS
allContents += concatContents.replace(/[^a-zA-Z0-9.-]+/gm, ''); //removing anything thats not alphamnumeric or . and -
allContents += concatContents.split(/(\%[^\%]+)/).map(part => { try { return decodeURIComponent(part) } catch(e) { return '' } }).join(''); //catch pedophile spammers url-fu with encoding
}
//global filters
hitGlobalFilter = globalSettings.filters.some(filter => { return allContents.includes(filter.toLowerCase()) });
, ban;
const [combinedString, strictCombinedString] = getFilterStrings(req, res, strictFiltering);
hitGlobalFilter = filters.some(filter => { return allContents.includes(filter.toLowerCase()) });
//block/ban edit
if (hitGlobalFilter) {
if (globalSettings.filterMode === 1) {
return dynamicResponse(req, res, 400, 'message', {
'title': 'Bad request',
'message': 'Your edit was blocked by a global word filter',
});
} else {
const banDate = new Date();
const banExpiry = new Date(globalSettings.filterBanDuration + banDate.getTime());
const ban = {
'ip': {
'cloak': res.locals.ip.cloak,
'raw': res.locals.ip.raw,
},
'type': res.locals.anonymizer ? 1 : 0,
'range': 0,
'reason': 'global word filter auto ban',
'board': null,
'posts': null,
'issuer': 'system', //what should i call this
'date': banDate,
'expireAt': banExpiry,
'allowAppeal': filterBanAppealable,
'showUser': true,
'seen': false
};
const insertedResult = await Bans.insertOne(ban);
ban._id = insertedResult.insertedId;
ban.ip.raw = null; //for dynamicresponse
return dynamicResponse(req, res, 403, 'ban', {
bans: [ban]
});
}
return filterActions(req, res, hitGlobalFilter, 0, globalFilterMode,
0, globalFilterBanDuration, null, filterBanAppealable, null);
}
}
}

70
models/forms/makepost.js
Unescape View File

@ -7,9 +7,11 @@ const path = require('path')
, uploadDirectory = require(__dirname+'/../../lib/file/uploaddirectory.js')
, Mongo = require(__dirname+'/../../db/db.js')
, Socketio = require(__dirname+'/../../lib/misc/socketio.js')
, { Stats, Posts, Boards, Files, Bans } = require(__dirname+'/../../db/')
, { Stats, Posts, Boards, Files } = require(__dirname+'/../../db/')
, cache = require(__dirname+'/../../lib/redis/redis.js')
, nameHandler = require(__dirname+'/../../lib/post/name.js')
, getFilterStrings = require(__dirname+'/../../lib/post/getfilterstrings.js')
, filterActions = require(__dirname+'/../../lib/post/filteractions.js')
, { prepareMarkdown } = require(__dirname+'/../../lib/post/markdown/markdown.js')
, messageHandler = require(__dirname+'/../../lib/post/message.js')
, moveUpload = require(__dirname+'/../../lib/file/moveupload.js')
@ -105,71 +107,39 @@ module.exports = async (req, res, next) => {
});
}
}
//filters
if (!res.locals.permissions.get(Permissions.BYPASS_FILTERS)) {
//deconstruct global filter settings to differnt names, else they would conflict with the respective board-level setting
const { filters: globalFilters, filterMode: globalFilterMode,
filterBanDuration: globalFilterBanDuration } = config.get;
let hitGlobalFilter = false
, hitLocalFilter = false
, ban;
let concatContents = `|${req.body.name}|${req.body.message}|${req.body.subject}|${req.body.email}|\
${res.locals.numFiles > 0 ? req.files.file.map(f => f.name+'|'+(f.phash || '')).join('|') : ''}`.toLowerCase();
let allContents = concatContents;
if (strictFiltering || res.locals.board.settings.strictFiltering) { //strict filtering adds a few transformations of the text to try and match filters when sers use techniques like zalgo, ZWS, markdown, multi-line, etc.
allContents += concatContents.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); //removing diacritics
allContents += concatContents.replace(/[\u200B-\u200D\uFEFF]/g, ''); //removing ZWS
allContents += concatContents.replace(/[^a-zA-Z0-9.-]+/gm, ''); //removing anything thats not alphamnumeric or . and -
allContents += concatContents.split(/(\%[^\%]+)/).map(part => { try { return decodeURIComponent(part) } catch(e) { return '' } }).join(''); //catch pedophile spammers url-fu with encoding
}
//global filters
let [combinedString, strictCombinedString] = getFilterStrings(req, res, strictFiltering || res.locals.board.settings.strictFiltering);
//compare to global filters
if (globalFilters && globalFilters.length > 0 && globalFilterMode > 0) {
hitGlobalFilter = globalFilters.some(filter => { return allContents.includes(filter.toLowerCase()) });
hitGlobalFilter = globalFilters.some(filter => { return strictCombinedString.includes(filter.toLowerCase()) });
}
//board-specific filters
//compare to board filters
if (!hitGlobalFilter && !res.locals.permissions.get(Permissions.MANAGE_BOARD_GENERAL)
&& filterMode > 0 && filters && filters.length > 0) {
const localFilterContents = res.locals.board.settings.strictFiltering ? allContents : concatContents;
const localFilterContents = res.locals.board.settings.strictFiltering === true ? strictCombinedString : combinedString;
hitLocalFilter = filters.some(filter => { return localFilterContents.includes(filter.toLowerCase()) });
}
//block post/apply bans if an active filter matched
if (hitGlobalFilter || hitLocalFilter) {
await deleteTempFiles(req).catch(e => console.error);
const useFilterMode = hitGlobalFilter ? globalFilterMode : filterMode; //global override local filter
if (useFilterMode === 1) {
return dynamicResponse(req, res, 400, 'message', {
'title': 'Bad request',
'message': 'Your post was blocked by a word filter',
'redirect': redirect
});
} else { //otherwise filter mode must be 2
const useFilterBanDuration = hitGlobalFilter ? globalFilterBanDuration : filterBanDuration;
const banBoard = hitGlobalFilter ? null : res.locals.board._id;
const banDate = new Date();
const banExpiry = new Date(useFilterBanDuration + banDate.getTime());
const ban = {
'ip': {
'cloak': res.locals.ip.cloak,
'raw': res.locals.ip.raw,
},
'type': res.locals.anonymizer ? 1 : 0,
'range': 0,
'reason': `${hitGlobalFilter ? 'global ' :''}word filter auto ban`,
'board': banBoard,
'posts': null,
'issuer': 'system', //what should i call this
'date': banDate,
'expireAt': banExpiry,
'allowAppeal': hitGlobalFilter ? filterBanAppealable : true,
'showUser': true,
'seen': false
};
const insertedResult = await Bans.insertOne(ban);
ban._id = insertedResult.insertedId;
ban.ip.raw = null; //for dynamicresponse
return dynamicResponse(req, res, 403, 'ban', {
bans: [ban]
});
}
return filterActions(req, res, hitGlobalFilter, filterMode, globalFilterMode,
filterBanDuration, globalFilterBanDuration, globalFilterBanDuration,
filterBanAppealable, redirect);
}
}
//for r9k messages. usually i wouldnt process these if its not enabled e.g. flags and IDs but in this case I think its necessary

Write Preview
Loading…
Cancel
Save