From 7ce421de3bc01460a64212595feb723a87dc48da Mon Sep 17 00:00:00 2001 From: disco Date: Fri, 2 Jun 2023 00:00:00 +0000 Subject: [PATCH] Redesign filter settings to allow multiple filters --- configs/template.js.example | 13 ---- controllers/forms.js | 19 +++++- controllers/forms/addfilter.js | 43 ++++++++++++++ controllers/forms/deletefilter.js | 39 ++++++++++++ controllers/forms/editfilter.js | 44 ++++++++++++++ controllers/forms/index.js | 3 + controllers/pages.js | 13 +++- db/filters.js | 64 ++++++++++++++++++++ db/index.js | 1 + gulp/res/css/style.css | 4 ++ gulpfile.js | 5 +- lib/post/checkfilters.js | 13 ++++ lib/post/filteractions.js | 23 ++++--- lib/post/getfilterstrings.js | 35 +++++------ models/forms/addfilter.js | 28 +++++++++ models/forms/changeboardsettings.js | 4 -- models/forms/changeglobalsettings.js | 9 --- models/forms/deleteboard.js | 3 +- models/forms/deletefilter.js | 26 ++++++++ models/forms/editfilter.js | 35 +++++++++++ models/forms/editpost.js | 23 ++++--- models/forms/makepost.js | 44 +++++++------- models/pages/globalmanage/editfilter.js | 26 ++++++++ models/pages/globalmanage/filters.js | 22 +++++++ models/pages/globalmanage/index.js | 2 + models/pages/manage/editfilter.js | 26 ++++++++ models/pages/manage/filters.js | 22 +++++++ models/pages/manage/index.js | 2 + test/board.js | 79 +++++++++++++++++++++++++ test/global.js | 78 ++++++++++++++++++++++++ test/pages.js | 1 + test/setup.js | 6 -- views/custompages/faq.pug | 7 +++ views/includes/filtereditform.pug | 28 +++++++++ views/includes/filternewform.pug | 29 +++++++++ views/mixins/filter.pug | 32 ++++++++++ views/mixins/globalmanagenav.pug | 3 + views/mixins/managenav.pug | 3 + views/pages/account.pug | 5 ++ views/pages/editfilter.pug | 18 ++++++ views/pages/globaleditfilter.pug | 18 ++++++ views/pages/globalmanagefilters.pug | 37 ++++++++++++ views/pages/globalmanagesettings.pug | 35 ----------- views/pages/managefilters.pug | 38 ++++++++++++ views/pages/managesettings.pug | 16 ----- 45 files changed, 867 insertions(+), 157 deletions(-) create mode 100644 controllers/forms/addfilter.js create mode 100644 controllers/forms/deletefilter.js create mode 100644 controllers/forms/editfilter.js create mode 100644 db/filters.js create mode 100644 lib/post/checkfilters.js create mode 100644 models/forms/addfilter.js create mode 100644 models/forms/deletefilter.js create mode 100644 models/forms/editfilter.js create mode 100644 models/pages/globalmanage/editfilter.js create mode 100644 models/pages/globalmanage/filters.js create mode 100644 models/pages/manage/editfilter.js create mode 100644 models/pages/manage/filters.js create mode 100644 views/includes/filtereditform.pug create mode 100644 views/includes/filternewform.pug create mode 100644 views/mixins/filter.pug create mode 100644 views/pages/editfilter.pug create mode 100644 views/pages/globaleditfilter.pug create mode 100644 views/pages/globalmanagefilters.pug create mode 100644 views/pages/managefilters.pug diff --git a/configs/template.js.example b/configs/template.js.example index 55dfefda..ba2bb5c8 100644 --- a/configs/template.js.example +++ b/configs/template.js.example @@ -26,12 +26,6 @@ module.exports = { language: 'en-GB', - filters: [], - strictFiltering: false, - filterMode: 0, - filterBanDuration: 0, - filterBanAppealable: true, - //settings for captchas captchaOptions: { type: 'text', @@ -177,9 +171,6 @@ module.exports = { //max number of quotes that will be linked in a post. 0 for unlimited (not recommended) quoteLimit: 25, - //aply global filters more aggressively, trying against extra text that strips diacritics and some ZWS chars - strictFiltering: true, - //how many replies to show on index pages under each OP previewReplies: 5, stickyPreviewReplies: 5, //choose a different amount for sticky posts if desired @@ -419,12 +410,8 @@ module.exports = { customCSS: null, blockedCountries: [], //2 char ISO country codes to block disableAnonymizerFilePosting: false, - filters: [], //words/phrases to block - filterMode: 0, //0=nothing, 1=prevent post, 2=auto ban - filterBanDuration: 0, //duration (in ms) to ban if filter mode=2 deleteProtectionAge: 0, //prevent non-staff OP from deleting their thread if it older than this age in ms deleteProtectionCount: 0, //prevent non-staff op deleting their thread if it has more than this many replies - strictFiltering: false, announcement: { raw: null, markdown: null diff --git a/controllers/forms.js b/controllers/forms.js index 9c753f32..71fc31d1 100644 --- a/controllers/forms.js +++ b/controllers/forms.js @@ -27,9 +27,10 @@ const express = require('express') editNewsController, deleteNewsController, uploadBannersController, deleteBannersController, addFlagsController, deleteFlagsController, boardSettingsController, transferController, addAssetsController, deleteAssetsController, resignController, deleteAccountController, loginController, registerController, changePasswordController, - deleteAccountsController, editAccountController, globalSettingsController, createBoardController, makePostController, - addStaffController, deleteStaffController, editStaffController, editCustomPageController, editPostController, - editRoleController, newCaptchaForm, blockBypassForm, logoutForm, deleteSessionsController } = require(__dirname+'/forms/index.js'); + deleteAccountsController, editAccountController, addFilterController, editFilterController, deleteFilterController, + globalSettingsController, createBoardController, makePostController, addStaffController, deleteStaffController, + editStaffController, editCustomPageController, editPostController, editRoleController, newCaptchaForm, + blockBypassForm, logoutForm, deleteSessionsController } = require(__dirname+'/forms/index.js'); //make new post router.post('/board/:board/post', geoIp, processIp, useSession, sessionRefresh, Boards.exists, setBoardLanguage, calcPerms, banCheck, fileMiddlewares.posts, @@ -58,6 +59,12 @@ router.post('/board/:board/settings', geoIp, processIp, useSession, sessionRefre hasPerms.one(Permissions.MANAGE_BOARD_SETTINGS), boardSettingsController.paramConverter, boardSettingsController.controller); router.post('/board/:board/editbans', useSession, sessionRefresh, csrf, Boards.exists, setBoardLanguage, calcPerms, isLoggedIn, hasPerms.one(Permissions.MANAGE_BOARD_BANS), editBansController.paramConverter, editBansController.controller); //edit bans +router.post('/board/:board/addfilter', useSession, sessionRefresh, csrf, Boards.exists, setBoardLanguage, calcPerms, isLoggedIn, + hasPerms.one(Permissions.MANAGE_BOARD_SETTINGS), addFilterController.paramConverter, addFilterController.controller); //add new filter +router.post('/board/:board/editfilter', useSession, sessionRefresh, csrf, Boards.exists, setBoardLanguage, calcPerms, isLoggedIn, + hasPerms.one(Permissions.MANAGE_BOARD_SETTINGS), editFilterController.paramConverter, editFilterController.controller); //edit filter +router.post('/board/:board/deletefilter', useSession, sessionRefresh, csrf, Boards.exists, setBoardLanguage, calcPerms, isLoggedIn, + hasPerms.one(Permissions.MANAGE_BOARD_SETTINGS), deleteFilterController.paramConverter, deleteFilterController.controller); //delete filter router.post('/board/:board/deleteboard', useSession, sessionRefresh, csrf, Boards.exists, setBoardLanguage, calcPerms, isLoggedIn, hasPerms.any(Permissions.MANAGE_BOARD_OWNER, Permissions.MANAGE_GLOBAL_BOARDS), deleteBoardController.controller); //delete board @@ -104,6 +111,12 @@ router.post('/global/editaccount', useSession, sessionRefresh, csrf, calcPerms, hasPerms.one(Permissions.MANAGE_GLOBAL_ACCOUNTS), editAccountController.paramConverter, editAccountController.controller); //account editing router.post('/global/editrole', useSession, sessionRefresh, csrf, calcPerms, isLoggedIn, hasPerms.one(Permissions.MANAGE_GLOBAL_ROLES), editRoleController.paramConverter, editRoleController.controller); //role editing +router.post('/global/addfilter', useSession, sessionRefresh, csrf, calcPerms, isLoggedIn, + hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), addFilterController.paramConverter, addFilterController.controller); //add new filter +router.post('/global/editfilter', useSession, sessionRefresh, csrf, calcPerms, isLoggedIn, + hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), editFilterController.paramConverter, editFilterController.controller); //edit filter +router.post('/global/deletefilter', useSession, sessionRefresh, csrf, calcPerms, isLoggedIn, + hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), deleteFilterController.paramConverter, deleteFilterController.controller); //delete filter router.post('/global/settings', useSession, sessionRefresh, csrf, calcPerms, isLoggedIn, hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), globalSettingsController.paramConverter, globalSettingsController.controller); //global settings diff --git a/controllers/forms/addfilter.js b/controllers/forms/addfilter.js new file mode 100644 index 00000000..f4b9a4fa --- /dev/null +++ b/controllers/forms/addfilter.js @@ -0,0 +1,43 @@ +'use strict'; + +const addFilter = require(__dirname+'/../../models/forms/addfilter.js') + , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js') + , paramConverter = require(__dirname+'/../../lib/middleware/input/paramconverter.js') + , { checkSchema, lengthBody, numberBody } = require(__dirname+'/../../lib/input/schema.js'); + +module.exports = { + + paramConverter: paramConverter({ + timeFields: ['filter_ban_duration'], + trimFields: ['filters', 'filter_message'], + numberFields: ['filter_mode'], + objectIdFields: ['filter_id'], + }), + + controller: async (req, res, next) => { + + const { __ } = res.locals; + + const errors = await checkSchema([ + { result: lengthBody(req.body.filters, 0, 50000), expected: false, error: __('Filter text cannot exceed 50000 characters') }, + { result: numberBody(req.body.filter_mode, 0, 2), expected: true, error: __('Filter mode must be a number from 0-2') }, + { result: numberBody(req.body.filter_ban_duration), expected: true, error: __('Invalid filter auto ban duration') }, + ]); + + if (errors.length > 0) { + return dynamicResponse(req, res, 400, 'message', { + 'title': __('Bad request'), + 'errors': req.params.board, + 'redirect': req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html' + }); + } + + try { + await addFilter(req, res, next); + } catch (err) { + return next(err); + } + + } + +}; diff --git a/controllers/forms/deletefilter.js b/controllers/forms/deletefilter.js new file mode 100644 index 00000000..51f88dd6 --- /dev/null +++ b/controllers/forms/deletefilter.js @@ -0,0 +1,39 @@ +'use strict'; + +const deleteFilter = require(__dirname+'/../../models/forms/deletefilter.js') + , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js') + , paramConverter = require(__dirname+'/../../lib/middleware/input/paramconverter.js') + , { checkSchema, lengthBody } = require(__dirname+'/../../lib/input/schema.js'); + +module.exports = { + + paramConverter: paramConverter({ + allowedArrays: ['checkedfilters'], + objectIdArrays: ['checkedfilters'] + }), + + controller: async (req, res, next) => { + + const { __ } = res.locals; + + const errors = await checkSchema([ + { result: lengthBody(req.body.checkedfilters, 1), expected: false, error: __('Must select at least one filter to delete') }, + ]); + + if (errors.length > 0) { + return dynamicResponse(req, res, 400, 'message', { + 'title': __('Bad request'), + 'errors': errors, + 'redirect': req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html' + }); + } + + try { + await deleteFilter(req, res, next); + } catch (err) { + return next(err); + } + + } + +}; diff --git a/controllers/forms/editfilter.js b/controllers/forms/editfilter.js new file mode 100644 index 00000000..90c3ec8e --- /dev/null +++ b/controllers/forms/editfilter.js @@ -0,0 +1,44 @@ +'use strict'; + +const editFilter = require(__dirname+'/../../models/forms/editfilter.js') + , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js') + , paramConverter = require(__dirname+'/../../lib/middleware/input/paramconverter.js') + , { checkSchema, lengthBody, numberBody, existsBody } = require(__dirname+'/../../lib/input/schema.js'); + +module.exports = { + + paramConverter: paramConverter({ + timeFields: ['filter_ban_duration'], + trimFields: ['filters', 'filter_message'], + numberFields: ['filter_mode'], + objectIdFields: ['filter_id'], + }), + + controller: async (req, res, next) => { + + const { __ } = res.locals; + + const errors = await checkSchema([ + { result: existsBody(req.body.filter_id), expected: true, error: __('Missing filter id') }, + { result: lengthBody(req.body.filters, 0, 50000), expected: false, error: __('Filter text cannot exceed 50000 characters') }, + { result: numberBody(req.body.filter_mode, 0, 2), expected: true, error: __('Filter mode must be a number from 0-2') }, + { result: numberBody(req.body.filter_ban_duration), expected: true, error: __('Invalid filter auto ban duration') }, + ]); + + if (errors.length > 0) { + return dynamicResponse(req, res, 400, 'message', { + 'title': __('Bad request'), + 'errors': errors, + 'redirect': req.headers.referer || (req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html') + }); + } + + try { + await editFilter(req, res, next); + } catch (err) { + return next(err); + } + + } + +}; diff --git a/controllers/forms/index.js b/controllers/forms/index.js index 598b5c68..6f2c743e 100644 --- a/controllers/forms/index.js +++ b/controllers/forms/index.js @@ -30,6 +30,9 @@ module.exports = { changePasswordController: require(__dirname+'/changepassword.js'), deleteSessionsController: require(__dirname+'/deletesessions.js'), deleteAccountsController: require(__dirname+'/deleteaccounts.js'), + addFilterController: require(__dirname+'/addfilter.js'), + editFilterController: require(__dirname+'/editfilter.js'), + deleteFilterController: require(__dirname+'/deletefilter.js'), globalSettingsController: require(__dirname+'/globalsettings.js'), createBoardController: require(__dirname+'/create.js'), makePostController: require(__dirname+'/makepost.js'), diff --git a/controllers/pages.js b/controllers/pages.js index 41a013d1..03063d06 100644 --- a/controllers/pages.js +++ b/controllers/pages.js @@ -18,15 +18,16 @@ const express = require('express') , setMinimal = require(__dirname+'/../lib/middleware/misc/setminimal.js') , { setBoardLanguage, setQueryLanguage } = require(__dirname+'/../lib/middleware/locale/locale.js') //page models - , { manageRecent, manageReports, manageAssets, manageSettings, manageBans, editCustomPage, manageMyPermissions, + , { manageRecent, manageReports, manageAssets, manageSettings, manageBans, manageFilters, editFilter, editCustomPage, manageMyPermissions, manageBoard, manageThread, manageLogs, manageCatalog, manageCustomPages, manageStaff, editStaff, editPost } = require(__dirname+'/../models/pages/manage/') - , { globalManageSettings, globalManageReports, globalManageBans, globalManageBoards, editNews, editAccount, editRole, + , { globalManageSettings, globalManageReports, globalManageBans, globalManageBoards, globalManageFilters, globalEditFilter, editNews, editAccount, editRole, globalManageRecent, globalManageAccounts, globalManageNews, globalManageLogs, globalManageRoles } = require(__dirname+'/../models/pages/globalmanage/') , { changePassword, blockBypass, home, register, login, create, myPermissions, sessions, setupTwoFactor, board, catalog, banners, boardSettings, globalSettings, randombanner, news, captchaPage, overboard, overboardCatalog, captcha, thread, modlog, modloglist, account, boardlist, customPage, csrfPage } = require(__dirname+'/../models/pages/') , threadParamConverter = paramConverter({ processThreadIdParam: true }) , logParamConverter = paramConverter({ processDateParam: true }) + , filterParamConverter = paramConverter({ objectIdParams: ['filterid'] }) , newsParamConverter = paramConverter({ objectIdParams: ['newsid'] }) , roleParamConverter = paramConverter({ objectIdParams: ['roleid'] }) , custompageParamConverter = paramConverter({ objectIdParams: ['custompageid'] }); @@ -87,6 +88,10 @@ router.get('/:board/manage/staff.html', useSession, sessionRefresh, isLoggedIn, hasPerms.one(Permissions.MANAGE_BOARD_STAFF), csrf, manageStaff); router.get('/:board/manage/editstaff/:staffusername([a-zA-Z0-9]{1,50}).html', useSession, sessionRefresh, isLoggedIn, Boards.exists, setBoardLanguage, calcPerms, hasPerms.one(Permissions.MANAGE_BOARD_STAFF), csrf, editStaff); +router.get('/:board/manage/filters.html', useSession, sessionRefresh, isLoggedIn, Boards.exists, setBoardLanguage, calcPerms, + hasPerms.one(Permissions.MANAGE_BOARD_SETTINGS), csrf, manageFilters); +router.get('/:board/manage/editfilter/:filterid([a-f0-9]{24}).html', useSession, sessionRefresh, isLoggedIn, Boards.exists, setBoardLanguage, calcPerms, + hasPerms.one(Permissions.MANAGE_BOARD_SETTINGS), csrf, filterParamConverter, editFilter); //global manage pages router.get('/globalmanage/reports.(html|json)', useSession, sessionRefresh, isLoggedIn, calcPerms, @@ -105,10 +110,14 @@ router.get('/globalmanage/accounts.html', useSession, sessionRefresh, isLoggedIn hasPerms.one(Permissions.MANAGE_GLOBAL_ACCOUNTS), csrf, globalManageAccounts); router.get('/globalmanage/roles.(html|json)', useSession, sessionRefresh, isLoggedIn, calcPerms, hasPerms.one(Permissions.MANAGE_GLOBAL_ROLES), csrf, globalManageRoles); +router.get('/globalmanage/filters.html', useSession, sessionRefresh, isLoggedIn, calcPerms, + hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), csrf, globalManageFilters); router.get('/globalmanage/settings.html', useSession, sessionRefresh, isLoggedIn, calcPerms, hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), csrf, globalManageSettings); router.get('/globalmanage/editnews/:newsid([a-f0-9]{24}).html', useSession, sessionRefresh, isLoggedIn, calcPerms, hasPerms.one(Permissions.MANAGE_GLOBAL_NEWS), csrf, newsParamConverter, editNews); +router.get('/globalmanage/editfilter/:filterid([a-f0-9]{24}).html', useSession, sessionRefresh, isLoggedIn, calcPerms, + hasPerms.one(Permissions.MANAGE_GLOBAL_SETTINGS), csrf, filterParamConverter, globalEditFilter); router.get('/globalmanage/editaccount/:accountusername([a-zA-Z0-9]{1,50}).html', useSession, sessionRefresh, isLoggedIn, calcPerms, hasPerms.one(Permissions.MANAGE_GLOBAL_ACCOUNTS), csrf, editAccount); router.get('/globalmanage/editrole/:roleid([a-f0-9]{24}).html', useSession, sessionRefresh, isLoggedIn, calcPerms, diff --git a/db/filters.js b/db/filters.js new file mode 100644 index 00000000..dc880aa2 --- /dev/null +++ b/db/filters.js @@ -0,0 +1,64 @@ + +'use strict'; + +const Mongo = require(__dirname+'/db.js') + , db = Mongo.db.collection('filters'); + +module.exports = { + + db, + + // null board retrieves global filters only + findForBoard: (board=null, limit=0) => { + return db.find({'board': board}).sort({ + '_id': -1 + }) + .limit(limit) + .toArray(); + }, + + findOne: (board, id) => { + return db.findOne({ + '_id': id, + 'board': board, + }); + }, + + updateOne: (board, id, filters, strictFiltering, filterMode, filterMessage, filterBanDuration, filterBanAppealable) => { + return db.updateOne({ + '_id': id, + 'board': board, + }, { + '$set': { + 'filters': filters, + 'strictFiltering': strictFiltering, + 'filterMode': filterMode, + 'filterMessage': filterMessage, + 'filterBanDuration': filterBanDuration, + 'filterBanAppealable': filterBanAppealable, + } + }); + }, + + insertOne: (filters) => { + return db.insertOne(filters); + }, + + deleteMany: (board, ids) => { + return db.deleteMany({ + '_id': { + '$in': ids + }, + 'board': board + }); + }, + + deleteBoard: (board) => { + return db.deleteMany({'board': board}); + }, + + deleteAll: () => { + return db.deleteMany({}); + }, + +}; diff --git a/db/index.js b/db/index.js index adb7bb1d..bb1a2912 100644 --- a/db/index.js +++ b/db/index.js @@ -11,6 +11,7 @@ module.exports = { Captchas: require(__dirname+'/captchas.js'), Files: require(__dirname+'/files.js'), News: require(__dirname+'/news.js'), + Filters: require(__dirname+'/filters.js'), CustomPages: require(__dirname+'/custompages.js'), Ratelimits: require(__dirname+'/ratelimits.js'), Modlogs: require(__dirname+'/modlogs.js'), diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index 3f3c688f..fc5319bc 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -1455,6 +1455,10 @@ table, .boardtable { text-align: center; } +.nowrap { + white-space: nowrap; +} + #settingsmodal{ min-width: 400px; } diff --git a/gulpfile.js b/gulpfile.js index 8b40068f..2c2da7db 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -161,7 +161,7 @@ async function wipe() { await Mongo.setConfig(defaultConfig); const collectionNames = ['accounts', 'bans', 'custompages', 'boards', 'captcha', 'files', - 'modlog','news', 'posts', 'poststats', 'ratelimit', 'bypass', 'roles']; + 'modlog', 'filters', 'news', 'posts', 'poststats', 'ratelimit', 'bypass', 'roles']; for (const name of collectionNames) { //drop collection so gulp reset can be run again. ignores error of dropping non existing collection first time await db.dropCollection(name).catch(() => {}); @@ -169,7 +169,7 @@ async function wipe() { } const { Boards, Posts, Captchas, Ratelimits, News, CustomPages, - Accounts, Files, Stats, Modlogs, Bans, Bypass, Roles } = require(__dirname+'/db/'); + Accounts, Files, Stats, Modlogs, Filters, Bans, Bypass, Roles } = require(__dirname+'/db/'); //wipe db shit await Promise.all([ @@ -186,6 +186,7 @@ async function wipe() { Modlogs.deleteAll(), Bypass.deleteAll(), News.deleteAll(), + Filters.deleteAll(), ]); //add indexes - should profiled and changed at some point if necessary diff --git a/lib/post/checkfilters.js b/lib/post/checkfilters.js new file mode 100644 index 00000000..daeec03c --- /dev/null +++ b/lib/post/checkfilters.js @@ -0,0 +1,13 @@ +'use strict'; + +module.exports = (filters, combinedString, strictCombinedString) => { + + for (const filter of filters) { + const string = filter.strictFiltering ? strictCombinedString : combinedString; + const hitFilter = filter.filters.find(match => { return string.includes(match.toLowerCase()); }); + if (hitFilter) { + return [ hitFilter, filter.filterMode, filter.filterMessage, filter.filterBanDuration, filter.filterBanAppealable ]; + } + } + return false; +}; diff --git a/lib/post/filteractions.js b/lib/post/filteractions.js index 03a94f5a..dbfe1f0d 100644 --- a/lib/post/filteractions.js +++ b/lib/post/filteractions.js @@ -4,24 +4,23 @@ const { Bans } = require(__dirname+'/../../db/') , dynamicResponse = require(__dirname+'/../misc/dynamic.js'); //ehhh, kinda too many args -module.exports = async (req, res, hitGlobalFilter, hitLocalFilter, boardFilterMode, globalFilterMode, - boardFilterBanDuration, globalFilterBanDuration, filterBanAppealable, redirect) => { +module.exports = async (req, res, globalFilter, hitFilter, filterMode, + filterMessage, filterBanDuration, filterBanAppealable, redirect) => { const { __ } = res.locals; - //global filter mode takes prio - const useFilterMode = hitGlobalFilter ? globalFilterMode : boardFilterMode; - if (useFilterMode === 1) { + const blockMessage = __('Your post was blocked by a word filter') + (filterMessage ? ': ' + filterMessage : ''); + + if (filterMode === 1) { return dynamicResponse(req, res, 400, 'message', { 'title': __('Bad request'), - 'message': __('Your post was blocked by a word filter'), + 'message': blockMessage, 'redirect': redirect }); } else { - const useFilterBanDuration = hitGlobalFilter ? globalFilterBanDuration : boardFilterBanDuration; - const banBoard = hitGlobalFilter ? null : res.locals.board._id; + const banBoard = globalFilter ? null : res.locals.board._id; const banDate = new Date(); - const banExpiry = new Date(useFilterBanDuration + banDate.getTime()); + const banExpiry = new Date(filterBanDuration + banDate.getTime()); const ban = { 'ip': { 'cloak': res.locals.ip.cloak, @@ -29,15 +28,15 @@ module.exports = async (req, res, hitGlobalFilter, hitLocalFilter, boardFilterMo 'type': res.locals.ip.type, }, 'range': 0, - 'reason': __(`${hitGlobalFilter ? 'global ' :''}word filter auto ban`), + 'reason': __(`${globalFilter ? 'global ' :''}word filter auto ban`) + (filterMessage ? ': ' + filterMessage : ''), 'board': banBoard, 'posts': null, 'issuer': 'system', //todo: make a "system" property instead? 'date': banDate, 'expireAt': banExpiry, - 'allowAppeal': hitGlobalFilter ? filterBanAppealable : true, + 'allowAppeal': filterBanAppealable, 'showUser': true, - 'note': __(`${hitGlobalFilter ? 'global ' :''}filter hit: "%s"`, (hitGlobalFilter || hitLocalFilter)), + 'note': __(`${globalFilter ? 'global ' :''}filter hit: "%s"`, (hitFilter)), 'seen': true, }; const insertedResult = await Bans.insertOne(ban); diff --git a/lib/post/getfilterstrings.js b/lib/post/getfilterstrings.js index b05902bd..212ef89c 100644 --- a/lib/post/getfilterstrings.js +++ b/lib/post/getfilterstrings.js @@ -1,34 +1,31 @@ 'use strict'; -module.exports = (req, res, strict=false) => { +module.exports = (req, res) => { //combines a bunch of parts of the post (name, subject, message, filenames+phashes) const fileStrings = res.locals.numFiles ? req.files.file.map(f => `${f.name}|${f.phash || ''}`).join('|') : 0; const combinedString = [req.body.name, req.body.message, req.body.subject, req.body.email, fileStrings].join('|').toLowerCase(); - let strictCombinedString = combinedString; //"strict" filtering adds a bunch of permutations to also compare filters with; - if (strict === true) { - - //diacritics and "zalgo" removed - strictCombinedString += combinedString.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); + let strictCombinedString = combinedString; - //zero width spaces removed - strictCombinedString += combinedString.replace(/[\u200B-\u200D\uFEFF]/g, ''); + //diacritics and "zalgo" removed + strictCombinedString += combinedString.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); - //just a-z, 0-9, . and - - strictCombinedString += combinedString.replace(/[^a-zA-Z0-9.-]+/gm, ''); + //zero width spaces removed + strictCombinedString += combinedString.replace(/[\u200B-\u200D\uFEFF]/g, ''); - //urlendoded characters in URLs replaced (todo: remove this if/when the url regex gets updated to no longer match these) - strictCombinedString += combinedString.split(/(%[^%]+)/).map(part => { - try { - return decodeURIComponent(part); - } catch(e) { - return ''; - } - }).join(''); + //just a-z, 0-9, . and - + strictCombinedString += combinedString.replace(/[^a-zA-Z0-9.-]+/gm, ''); - } + //urlendoded characters in URLs replaced (todo: remove this if/when the url regex gets updated to no longer match these) + strictCombinedString += combinedString.split(/(%[^%]+)/).map(part => { + try { + return decodeURIComponent(part); + } catch(e) { + return ''; + } + }).join(''); return { combinedString, strictCombinedString }; diff --git a/models/forms/addfilter.js b/models/forms/addfilter.js new file mode 100644 index 00000000..f4b49c21 --- /dev/null +++ b/models/forms/addfilter.js @@ -0,0 +1,28 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../db/') + , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js'); + +module.exports = async (req, res) => { + + const { __ } = res.locals; + + const filter = { + 'board': req.params.board ? req.params.board : null, + 'filters': req.body.filters.split(/\r?\n/).filter(n => n), + 'strictFiltering': req.body.strict_filtering ? true : false, + 'filterMode': req.body.filter_mode, + 'filterMessage': req.body.filter_message, + 'filterBanDuration': req.body.filter_ban_duration, + 'filterBanAppealable': req.body.filter_ban_appealable ? true : false, + }; + + await Filters.insertOne(filter); + + return dynamicResponse(req, res, 200, 'message', { + 'title': __('Success'), + 'message': __('Added filter'), + 'redirect': req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html' + }); + +}; diff --git a/models/forms/changeboardsettings.js b/models/forms/changeboardsettings.js index 4f7f5876..3fa1a91d 100644 --- a/models/forms/changeboardsettings.js +++ b/models/forms/changeboardsettings.js @@ -103,14 +103,10 @@ module.exports = async (req, res) => { 'lockMode': numberSetting(req.body.lock_mode, oldSettings.lockMode), 'messageR9KMode': numberSetting(req.body.message_r9k_mode, oldSettings.messageR9KMode), 'fileR9KMode': numberSetting(req.body.file_r9k_mode, oldSettings.fileR9KMode), - 'filterMode': numberSetting(req.body.filter_mode, oldSettings.filterMode), - 'filterBanDuration': numberSetting(req.body.ban_duration, oldSettings.filterBanDuration), 'deleteProtectionAge': numberSetting(req.body.delete_protection_age, oldSettings.deleteProtectionAge), 'deleteProtectionCount': numberSetting(req.body.delete_protection_count, oldSettings.deleteProtectionCount), - 'filters': arraySetting(req.body.filters, oldSettings.filters, 50), 'blockedCountries': req.body.countries || [], 'disableAnonymizerFilePosting': booleanSetting(req.body.disable_anonymizer_file_posting), - 'strictFiltering': booleanSetting(req.body.strict_filtering), 'customCss': globalLimits.customCss.enabled ? (req.body.custom_css !== null ? req.body.custom_css : oldSettings.customCss) : null, 'announcement': { 'raw': announcement !== null ? announcement : oldSettings.announcement.raw, diff --git a/models/forms/changeglobalsettings.js b/models/forms/changeglobalsettings.js index 7c206e24..d314c94a 100644 --- a/models/forms/changeglobalsettings.js +++ b/models/forms/changeglobalsettings.js @@ -51,11 +51,6 @@ module.exports = async (req, res) => { } const newSettings = { - filters: arraySetting(req.body.filters, oldSettings.filters), - filterMode: numberSetting(req.body.filter_mode, oldSettings.filterMode), - strictFiltering: booleanSetting(req.body.strict_filtering, oldSettings.strictFiltering), - filterBanDuration: numberSetting(req.body.ban_duration, oldSettings.filterBanDuration), - filterBanAppealable: booleanSetting(req.body.filter_ban_appealable), allowedHosts: arraySetting(req.body.allowed_hosts, oldSettings.allowedHosts), countryCodeHeader: trimSetting(req.body.country_code_header, oldSettings.countryCodeHeader), ipHeader: trimSetting(req.body.ip_header, oldSettings.ipHeader), @@ -318,14 +313,10 @@ module.exports = async (req, res) => { maxThreadMessageLength: numberSetting(req.body.board_defaults_max_thread_message_length, oldSettings.boardDefaults.maxThreadMessageLength), maxReplyMessageLength: numberSetting(req.body.board_defaults_max_reply_message_length, oldSettings.boardDefaults.maxReplyMessageLength), disableAnonymizerFilePosting: booleanSetting(req.body.board_defaults_disable_anonymizer_file_posting, oldSettings.boardDefaults.disableAnonymizerFilePosting), - filterMode: numberSetting(req.body.board_defaults_filter_mode, oldSettings.boardDefaults.filterMode), - filterBanDuration: numberSetting(req.body.board_defaults_filter_ban_duration, oldSettings.boardDefaults.filterBanDuration), deleteProtectionAge: numberSetting(req.body.board_defaults_delete_protection_age, oldSettings.boardDefaults.deleteProtectionAge), deleteProtectionCount: numberSetting(req.body.board_defaults_delete_protection_count, oldSettings.boardDefaults.deleteProtectionCount), - strictFiltering: booleanSetting(req.body.board_defaults_strict_filtering, oldSettings.boardDefaults.strictFiltering), customCSS: null, blockedCountries: [], - filters: [], announcement: { raw: null, markdown: null diff --git a/models/forms/deleteboard.js b/models/forms/deleteboard.js index 19facafa..56e9a11d 100644 --- a/models/forms/deleteboard.js +++ b/models/forms/deleteboard.js @@ -1,6 +1,6 @@ 'use strict'; -const { CustomPages, Accounts, Boards, Stats, Posts, Bans, Modlogs } = require(__dirname+'/../../db/') +const { CustomPages, Accounts, Boards, Stats, Posts, Bans, Modlogs, Filters } = require(__dirname+'/../../db/') , deletePosts = require(__dirname+'/deletepost.js') , uploadDirectory = require(__dirname+'/../../lib/file/uploaddirectory.js') , i18n = require(__dirname+'/../../lib/locale/locale.js') @@ -27,6 +27,7 @@ module.exports = async (uri, board) => { Object.keys(board.staff).length > 0 ? Accounts.removeStaffBoard(Object.keys(board.staff), uri) : void 0, //remove staffboard from staff accounts Modlogs.deleteBoard(uri), //modlogs for the board Bans.deleteBoard(uri), //bans for the board + Filters.deleteBoard(uri), //filters for the board Stats.deleteBoard(uri), //stats for the board CustomPages.deleteBoard(uri), //custom pages for the board remove(`${uploadDirectory}/html/${uri}/`), //html diff --git a/models/forms/deletefilter.js b/models/forms/deletefilter.js new file mode 100644 index 00000000..d5d4d821 --- /dev/null +++ b/models/forms/deletefilter.js @@ -0,0 +1,26 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../db/') + , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js'); + +module.exports = async (req, res) => { + + const { __, __n } = res.locals; + + const deletedFilters = await Filters.deleteMany(req.params.board, req.body.checkedfilters).then(result => result.deletedCount); + + if (deletedFilters === 0 || deletedFilters < req.body.checkedfilters.length) { + return dynamicResponse(req, res, 400, 'message', { + 'title': __('Bad request'), + 'error': __n('Deleted %s filters', deletedFilters), + 'redirect': req.headers.referer || (req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html') + }); + } + + return dynamicResponse(req, res, 200, 'message', { + 'title': __('Success'), + 'message': __n('Deleted %s filters', deletedFilters), + 'redirect': req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html' + }); + +}; diff --git a/models/forms/editfilter.js b/models/forms/editfilter.js new file mode 100644 index 00000000..8d3f5bf2 --- /dev/null +++ b/models/forms/editfilter.js @@ -0,0 +1,35 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../db/') + , dynamicResponse = require(__dirname+'/../../lib/misc/dynamic.js'); + +module.exports = async (req, res) => { + + const { __ } = res.locals; + + const updated = await Filters.updateOne( + req.params.board, + req.body.filter_id, + req.body.filters.split(/\r?\n/).filter(n => n), + req.body.strict_filtering ? true : false, + req.body.filter_mode, + req.body.filter_message, + req.body.filter_ban_duration, + req.body.filter_ban_appealable ? true : false + ).then(r => r.matchedCount); + + if (updated === 0) { + return dynamicResponse(req, res, 400, 'message', { + 'title': __('Bad request'), + 'error': __('Filter does not exist'), + 'redirect': req.headers.referer || (req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html') + }); + } + + return dynamicResponse(req, res, 200, 'message', { + 'title': __('Success'), + 'message': __('Updated filter'), + 'redirect': req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html' + }); + +}; diff --git a/models/forms/editpost.js b/models/forms/editpost.js index 358ae49b..0aa7e646 100644 --- a/models/forms/editpost.js +++ b/models/forms/editpost.js @@ -1,6 +1,6 @@ 'use strict'; -const { Posts, Modlogs } = require(__dirname+'/../../db/') +const { Posts, Modlogs, Filters } = require(__dirname+'/../../db/') , { Permissions } = require(__dirname+'/../../lib/permission/permissions.js') , { createHash } = require('crypto') , Mongo = require(__dirname+'/../../db/db.js') @@ -8,6 +8,7 @@ const { Posts, Modlogs } = require(__dirname+'/../../db/') , messageHandler = require(__dirname+'/../../lib/post/message.js') , nameHandler = require(__dirname+'/../../lib/post/name.js') , getFilterStrings = require(__dirname+'/../../lib/post/getfilterstrings.js') + , checkFilters = require(__dirname+'/../../lib/post/checkfilters.js') , filterActions = require(__dirname+'/../../lib/post/filteractions.js') , ModlogActions = require(__dirname+'/../../lib/input/modlogactions.js') , config = require(__dirname+'/../../lib/misc/config.js') @@ -25,22 +26,20 @@ todo: handle some more situations */ const { __ } = res.locals; - const { filterBanAppealable, previewReplies, strictFiltering } = config.get; + const { previewReplies } = config.get; const { board, post } = res.locals; //filters if (!res.locals.permissions.get(Permissions.BYPASS_FILTERS)) { //only global filters are checked, because anybody who could edit bypasses board filters - const { filters, filterMode, filterBanDuration } = config.get; - if (filters.length > 0 && filterMode > 0) { - let hitGlobalFilter = false; - const { strictCombinedString } = getFilterStrings(req, res, strictFiltering); - hitGlobalFilter = filters.find(filter => { return strictCombinedString.includes(filter.toLowerCase()); }); - //block/ban edit - if (hitGlobalFilter) { - return filterActions(req, res, hitGlobalFilter, null, 0, filterMode, - 0, filterBanDuration, filterBanAppealable, null); - } + const globalFilters = await Filters.findForBoard(null); + + let hitFilter = false; + let { combinedString, strictCombinedString } = getFilterStrings(req, res); + + hitFilter = checkFilters(globalFilters, combinedString, strictCombinedString); + if (hitFilter) { + return filterActions(req, res, true, hitFilter[0], hitFilter[1], hitFilter[2], hitFilter[3], hitFilter[4], null); } } diff --git a/models/forms/makepost.js b/models/forms/makepost.js index 72f6ff80..21d81135 100644 --- a/models/forms/makepost.js +++ b/models/forms/makepost.js @@ -6,10 +6,11 @@ const { createHash, randomBytes } = require('crypto') , uploadDirectory = require(__dirname+'/../../lib/file/uploaddirectory.js') , Mongo = require(__dirname+'/../../db/db.js') , Socketio = require(__dirname+'/../../lib/misc/socketio.js') - , { Stats, Posts, Boards, Files } = require(__dirname+'/../../db/') + , { Stats, Posts, Boards, Files, Filters } = require(__dirname+'/../../db/') , cache = require(__dirname+'/../../lib/redis/redis.js') , nameHandler = require(__dirname+'/../../lib/post/name.js') , getFilterStrings = require(__dirname+'/../../lib/post/getfilterstrings.js') + , checkFilters = require(__dirname+'/../../lib/post/checkfilters.js') , filterActions = require(__dirname+'/../../lib/post/filteractions.js') , { prepareMarkdown } = require(__dirname+'/../../lib/post/markdown/markdown.js') , messageHandler = require(__dirname+'/../../lib/post/message.js') @@ -37,8 +38,8 @@ const { createHash, randomBytes } = require('crypto') module.exports = async (req, res) => { const { __ } = res.locals; - const { filterBanAppealable, checkRealMimeTypes, thumbSize, thumbExtension, videoThumbPercentage, - strictFiltering, audioThumbnails, dontStoreRawIps, globalLimits } = config.get; + const { checkRealMimeTypes, thumbSize, thumbExtension, videoThumbPercentage, audioThumbnails, + dontStoreRawIps, globalLimits } = config.get; //spam/flood check const flood = await spamCheck(req, res); @@ -56,7 +57,7 @@ module.exports = async (req, res) => { let salt = null; let thread = null; const isStaffOrGlobal = res.locals.permissions.hasAny(Permissions.MANAGE_GLOBAL_GENERAL, Permissions.MANAGE_BOARD_GENERAL); - const { filterBanDuration, filterMode, filters, blockedCountries, threadLimit, ids, userPostSpoiler, + const { blockedCountries, threadLimit, ids, userPostSpoiler, pphTrigger, tphTrigger, tphTriggerAction, pphTriggerAction, sageOnlyEmail, forceAnon, replyLimit, disableReplySubject, captchaMode, lockMode, allowedFileTypes, customFlags, geoFlags, fileR9KMode, messageR9KMode } = res.locals.board.settings; @@ -114,32 +115,29 @@ module.exports = async (req, res) => { if (!res.locals.permissions.get(Permissions.BYPASS_FILTERS)) { //deconstruct global filter settings to differnt names, else they would conflict with the respective board-level setting - const { filters: globalFilters, filterMode: globalFilterMode, - filterBanDuration: globalFilterBanDuration } = config.get; + const [globalFilters, localFilters] = await Promise.all([ + Filters.findForBoard(null), + Filters.findForBoard(res.locals.board._id) + ]); - let hitGlobalFilter = false - , hitLocalFilter = false; - let { combinedString, strictCombinedString } = getFilterStrings(req, res, strictFiltering || res.locals.board.settings.strictFiltering); + let hitFilter = false + , globalFilter = false; + let { combinedString, strictCombinedString } = getFilterStrings(req, res); //compare to global filters - if (globalFilters && globalFilters.length > 0 && globalFilterMode > 0) { - hitGlobalFilter = globalFilters.find(filter => { return strictCombinedString.includes(filter.toLowerCase()); }); + hitFilter = checkFilters(globalFilters, combinedString, strictCombinedString); + if (hitFilter) { + globalFilter = true; } - - //compare to board filters - if (!hitGlobalFilter && !res.locals.permissions.get(Permissions.MANAGE_BOARD_GENERAL) - && filterMode > 0 && filters && filters.length > 0) { - const localFilterContents = res.locals.board.settings.strictFiltering === true ? strictCombinedString : combinedString; - hitLocalFilter = filters.find(filter => { return localFilterContents.includes(filter.toLowerCase()); }); + //if none matched, check local filters + if (!hitFilter) { + hitFilter = checkFilters(localFilters, combinedString, strictCombinedString); } - //block post/apply bans if an active filter matched - if (hitGlobalFilter || hitLocalFilter) { + if (hitFilter) { await deleteTempFiles(req).catch(console.error); - return filterActions(req, res, hitGlobalFilter, hitLocalFilter, filterMode, globalFilterMode, - filterBanDuration, globalFilterBanDuration, filterBanAppealable, redirect); + return filterActions(req, res, globalFilter, hitFilter[0], hitFilter[1], hitFilter[2], hitFilter[3], hitFilter[4], redirect); } - } //for r9k messages. usually i wouldnt process these if its not enabled e.g. flags and IDs but in this case I think its necessary @@ -295,7 +293,7 @@ module.exports = async (req, res) => { const videoStreams = audioVideoData.streams.filter(stream => stream.width != null); //filter to only video streams or something with a resolution if (videoStreams.length > 0) { processedFile.thumbextension = thumbExtension; - processedFile.geometry = {width: videoStreams[0].width, height: videoStreams[0].height}; + processedFile.geometry = {width: videoStreams[0].coded_width, height: videoStreams[0].coded_height}; if (Math.floor(processedFile.geometry.width*processedFile.geometry.height) > globalLimits.postFilesSize.videoResolution) { await deleteTempFiles(req).catch(console.error); return dynamicResponse(req, res, 400, 'message', { diff --git a/models/pages/globalmanage/editfilter.js b/models/pages/globalmanage/editfilter.js new file mode 100644 index 00000000..008c150b --- /dev/null +++ b/models/pages/globalmanage/editfilter.js @@ -0,0 +1,26 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../../db/'); + +module.exports = async (req, res, next) => { + + let filter; + try { + filter = await Filters.findOne(null, req.params.filterid); + } catch (err) { + return next(err); + } + + if (!filter) { + return next(); + } + + res + .set('Cache-Control', 'private, max-age=5') + .render('globaleditfilter', { + csrf: req.csrfToken(), + permissions: res.locals.permissions, + filter, + }); + +}; diff --git a/models/pages/globalmanage/filters.js b/models/pages/globalmanage/filters.js new file mode 100644 index 00000000..ad658056 --- /dev/null +++ b/models/pages/globalmanage/filters.js @@ -0,0 +1,22 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../../db/'); + +module.exports = async (req, res, next) => { + + let filters; + try { + filters = await Filters.findForBoard(null); + } catch (err) { + return next(err); + } + + res + .set('Cache-Control', 'private, max-age=5') + .render('globalmanagefilters', { + csrf: req.csrfToken(), + permissions: res.locals.permissions, + filters, + }); + +}; diff --git a/models/pages/globalmanage/index.js b/models/pages/globalmanage/index.js index c9a35904..8b16f272 100644 --- a/models/pages/globalmanage/index.js +++ b/models/pages/globalmanage/index.js @@ -7,9 +7,11 @@ module.exports = { globalManageBoards: require(__dirname+'/boards.js'), globalManageRecent: require(__dirname+'/recent.js'), globalManageNews: require(__dirname+'/news.js'), + globalManageFilters: require(__dirname+'/filters.js'), globalManageAccounts: require(__dirname+'/accounts.js'), globalManageSettings: require(__dirname+'/settings.js'), globalManageRoles: require(__dirname+'/roles.js'), + globalEditFilter: require(__dirname+'/editfilter.js'), editNews: require(__dirname+'/editnews.js'), editAccount: require(__dirname+'/editaccount.js'), editRole: require(__dirname+'/editrole.js'), diff --git a/models/pages/manage/editfilter.js b/models/pages/manage/editfilter.js new file mode 100644 index 00000000..87e66e27 --- /dev/null +++ b/models/pages/manage/editfilter.js @@ -0,0 +1,26 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../../db/'); + +module.exports = async (req, res, next) => { + + let filter; + try { + filter = await Filters.findOne(req.params.board, req.params.filterid); + } catch (err) { + return next(err); + } + + if (!filter) { + return next(); + } + + res + .set('Cache-Control', 'private, max-age=5') + .render('editfilter', { + csrf: req.csrfToken(), + permissions: res.locals.permissions, + filter, + }); + +}; diff --git a/models/pages/manage/filters.js b/models/pages/manage/filters.js new file mode 100644 index 00000000..6f766527 --- /dev/null +++ b/models/pages/manage/filters.js @@ -0,0 +1,22 @@ +'use strict'; + +const { Filters } = require(__dirname+'/../../../db/'); + +module.exports = async (req, res, next) => { + + let filters; + try { + filters = await Filters.findForBoard(req.params.board); + } catch (err) { + return next(err); + } + + res + .set('Cache-Control', 'private, max-age=5') + .render('managefilters', { + csrf: req.csrfToken(), + permissions: res.locals.permissions, + filters, + }); + +}; diff --git a/models/pages/manage/index.js b/models/pages/manage/index.js index edfb9486..4c90e633 100644 --- a/models/pages/manage/index.js +++ b/models/pages/manage/index.js @@ -4,6 +4,7 @@ module.exports = { manageReports: require(__dirname+'/reports.js'), manageRecent: require(__dirname+'/recent.js'), manageSettings: require(__dirname+'/settings.js'), + manageFilters: require(__dirname+'/filters.js'), manageBans: require(__dirname+'/bans.js'), manageLogs: require(__dirname+'/logs.js'), manageAssets: require(__dirname+'/assets.js'), @@ -13,6 +14,7 @@ module.exports = { manageCustomPages: require(__dirname+'/custompages.js'), manageMyPermissions: require(__dirname+'/mypermissions.js'), editCustomPage: require(__dirname+'/editcustompage.js'), + editFilter: require(__dirname+'/editfilter.js'), editPost: require(__dirname+'/editpost.js'), manageStaff: require(__dirname+'/staff.js'), editStaff: require(__dirname+'/editstaff.js'), diff --git a/test/board.js b/test/board.js index ed4d1731..f9cbb2d3 100644 --- a/test/board.js +++ b/test/board.js @@ -230,6 +230,85 @@ testing 123` expect(response2.status).toBe(404); }); + let filterId; + test('add filter post', async () => { + const params = new URLSearchParams({ + _csrf: csrfToken, + filters: `notgood +bad words`, + strict_filtering: 'true', + filter_mode: '1', + filter_message: 'Rule+1:+No+fun+allowed', + filter_ban_duration: '1s', + filter_ban_appealable: 'true', + }); + const response = await fetch('http://localhost/forms/board/test/addfilter', { + headers: { + 'x-using-xhr': 'true', + 'cookie': sessionCookie, + }, + method: 'POST', + body: params, + redirect: 'manual', + }); + expect(response.ok).toBe(true); + const filterPage = await fetch('http://localhost/test/manage/filter.html', { + headers: { + 'cookie': sessionCookie, + }, + }).then(res => res.text()); + const checkIndex = filterPage.indexOf('name="checkedfilter" value="'); + filterId = filterPage.substring(checkIndex+28, checkIndex+28+24); + }); + + test('edit filter post', async () => { + const params = new URLSearchParams({ + _csrf: csrfToken, + board: test, + filter_id: filterId, + filters: 'edited filters', + strict_filtering: 'true', + filter_mode: '0', + filter_message: 'edited message', + filter_ban_duration: '0' + // filter_ban_appealable omitted to change to false + }); + const response = await fetch('http://localhost/forms/board/test/editfilter', { + headers: { + 'x-using-xhr': 'true', + 'cookie': sessionCookie, + }, + method: 'POST', + body: params, + redirect: 'manual', + }); + expect(response.ok).toBe(true); + const filterPage = await fetch('http://localhost/test/manage/filter.html', { + headers: { + 'cookie': sessionCookie, + }, + }).then(res => res.text()); + const editTextIndex = filterPage.indexOf('edited filters'); + expect(editTextIndex).not.toBe(-1); + }); + + test('delete filter post', async () => { + const params = new URLSearchParams({ + _csrf: csrfToken, + checkedfilter: filterId, + }); + const response = await fetch('http://localhost/forms/board/test/deletefilter', { + headers: { + 'x-using-xhr': 'true', + 'cookie': sessionCookie, + }, + method: 'POST', + body: params, + redirect: 'manual', + }); + expect(response.ok).toBe(true); + }); + test('add staff', async () => { const params = new URLSearchParams({ _csrf: csrfToken, diff --git a/test/global.js b/test/global.js index 1f8b11a9..027760ff 100644 --- a/test/global.js +++ b/test/global.js @@ -98,6 +98,84 @@ testing 123` expect(response.ok).toBe(true); }); + let filterId; + test('add filter post', async () => { + const params = new URLSearchParams({ + _csrf: csrfToken, + filters: `notgood +bad words`, + strict_filtering: 'true', + filter_mode: '1', + filter_message: 'Rule+1:+No+fun+allowed', + filter_ban_duration: '1s', + filter_ban_appealable: 'true', + }); + const response = await fetch('http://localhost/forms/global/addfilter', { + headers: { + 'x-using-xhr': 'true', + 'cookie': sessionCookie, + }, + method: 'POST', + body: params, + redirect: 'manual', + }); + expect(response.ok).toBe(true); + const filterPage = await fetch('http://localhost/globalmanage/filter.html', { + headers: { + 'cookie': sessionCookie, + }, + }).then(res => res.text()); + const checkIndex = filterPage.indexOf('name="checkedfilter" value="'); + filterId = filterPage.substring(checkIndex+28, checkIndex+28+24); + }); + + test('edit filter post', async () => { + const params = new URLSearchParams({ + _csrf: csrfToken, + filter_id: filterId, + filters: 'edited filters', + strict_filtering: 'true', + filter_mode: '0', + filter_message: 'edited message', + filter_ban_duration: '0' + // filter_ban_appealable omitted to change to false + }); + const response = await fetch('http://localhost/forms/global/editfilter', { + headers: { + 'x-using-xhr': 'true', + 'cookie': sessionCookie, + }, + method: 'POST', + body: params, + redirect: 'manual', + }); + expect(response.ok).toBe(true); + const filterPage = await fetch('http://localhost/globalmanage/filter.html', { + headers: { + 'cookie': sessionCookie, + }, + }).then(res => res.text()); + const editTextIndex = filterPage.indexOf('edited filters'); + expect(editTextIndex).not.toBe(-1); + }); + + test('delete filter post', async () => { + const params = new URLSearchParams({ + _csrf: csrfToken, + checkedfilter: filterId, + }); + const response = await fetch('http://localhost/forms/global/deletefilter', { + headers: { + 'x-using-xhr': 'true', + 'cookie': sessionCookie, + }, + method: 'POST', + body: params, + redirect: 'manual', + }); + expect(response.ok).toBe(true); + }); + test('register test account', async () => { const params = new URLSearchParams({ _csrf: csrfToken, diff --git a/test/pages.js b/test/pages.js index 3d9c3a23..ecf54d2f 100644 --- a/test/pages.js +++ b/test/pages.js @@ -27,6 +27,7 @@ module.exports = () => describe('Test loading a bunch of pages', () => { 'globalmanage/roles.html', 'globalmanage/roles.json', 'globalmanage/news.html', + 'globalmanage/filters.html', 'globalmanage/settings.html', ]; diff --git a/test/setup.js b/test/setup.js index f2a68b0a..6bbbe761 100644 --- a/test/setup.js +++ b/test/setup.js @@ -138,10 +138,6 @@ module.exports = () => describe('login and create test board', () => { block_bypass_force_anonymizers: 'true', block_bypass_expire_after_uses: '50', block_bypass_expire_after_time: '86400000', - filters: '', - strict_filtering: 'true', - filter_mode: '0', - ban_duration: '0', flood_timers_same_content_same_ip: '0', flood_timers_same_content_any_ip: '0', flood_timers_any_content_same_ip: '0', @@ -267,8 +263,6 @@ module.exports = () => describe('login and create test board', () => { board_defaults_max_reply_message_length: '20000', board_defaults_delete_protection_count: '0', board_defaults_delete_protection_age: '0', - board_defaults_filter_mode: '0', - board_defaults_filter_ban_duration: '0', board_defaults_allowed_file_types_video: 'true', board_defaults_allowed_file_types_image: 'true', board_defaults_allowed_file_types_animated_image: 'true', diff --git a/views/custompages/faq.pug b/views/custompages/faq.pug index 898f9bb4..d1220d0a 100644 --- a/views/custompages/faq.pug +++ b/views/custompages/faq.pug @@ -262,6 +262,13 @@ block content p Early 404: When a new thread is posted, delete any existing threads with less than #{early404Replies} replies beyond the first 1/#{early404Fraction} of threads. p Disable anonymizer file posting: Prevent users posting images through anonymizers such as Tor hidden services, lokinet SNApps or i2p eepsites. p Blocked Countries: Block country codes (based on geo Ip data) from posting. + .table-container.flex-center.mv-5 + .anchor#filters + table + tr + th: a(href='#filters') What do the filter options do? + tr + td p Filters: Newline separated list of words or phrases to match in posts. Checks name, message, email, subject, and filenames. p Strict Filtering: More aggressively match filters, by normalising the input compared against the filters. p Filter Mode: What to do when a post matches a filter. diff --git a/views/includes/filtereditform.pug b/views/includes/filtereditform.pug new file mode 100644 index 00000000..71979ab0 --- /dev/null +++ b/views/includes/filtereditform.pug @@ -0,0 +1,28 @@ +input(type='hidden' name='_csrf' value=csrf) +input(type='hidden' name='filter_id' value=filter._id) +.table-container.flex-center.mv-5 + table + tr + th #{__('Filters')} + th #{__('Strict Filtering')} + th #{__('Filter Mode')} + th #{__('Block/Ban Message')} + th #{__('Filter Auto Ban Duration')} + th #{__('Filter Bans Appealable')} + tr + td + textarea(name='filters' rows='2' placeholder=__('Newline separated') required) #{filter.filters.join('\n')} + td.text-center + input(type='checkbox', name='strict_filtering', value='true', checked=filter.strictFiltering) + td + select(name='filter_mode') + option(value='0', selected=filter.filterMode === 0) #{__('Do nothing')} + option(value='1', selected=filter.filterMode === 1) #{__('Block post')} + option(value='2', selected=filter.filterMode === 2) #{__('Ban')} + td + input(type='text' name='filter_message' placeholder=__('e.g. Rule 1: No ad spam') value=filter.filterMessage) + td + input(type='text' name='filter_ban_duration' placeholder=__('e.g. 3d') value=filter.filterBanDuration) + td.text-center + input(type='checkbox', name='filter_ban_appealable', value='true', checked=filter.filterBanAppealable) +input(type='submit', value=__('Save')) \ No newline at end of file diff --git a/views/includes/filternewform.pug b/views/includes/filternewform.pug new file mode 100644 index 00000000..a4027596 --- /dev/null +++ b/views/includes/filternewform.pug @@ -0,0 +1,29 @@ +input(type='hidden' name='_csrf' value=csrf) +.table-container.flex-center.mv-5 + table + tr + th #{__('Filters')} + th #{__('Strict Filtering')} + th #{__('Filter Mode')} + th #{__('Block/Ban Message')} + th #{__('Filter Auto Ban Duration')} + th #{__('Filter Bans Appealable')} + tr + td + textarea(name='filters' rows='2' placeholder=__('Newline separated') required) + td.text-center + label + input(type='checkbox', name='strict_filtering', value='true') + td + select(name='filter_mode') + option(value='0', selected=true) #{__('Do nothing')} + option(value='1') #{__('Block post')} + option(value='2') #{__('Ban')} + td + input(type='text' name='filter_message' placeholder=__('e.g. Rule 1: No ad spam')) + td + input(type='text' name='filter_ban_duration' placeholder=__('e.g. 3d')) + td.text-center + label + input(type='checkbox', name='filter_ban_appealable', value='true') +input(type='submit', value=__('Submit')) \ No newline at end of file diff --git a/views/mixins/filter.pug b/views/mixins/filter.pug new file mode 100644 index 00000000..bc51efaf --- /dev/null +++ b/views/mixins/filter.pug @@ -0,0 +1,32 @@ +mixin filter(f, globalmanage=false) + .anchor(id=f._id) + tr + td + input.left.post-check(type='checkbox', name='checkedfilters' value=f._id) + td + each line in f.filters + pre.no-m-p.nowrap !{line} + td.text-center + label + input(type='checkbox', name='strict_filtering', value='true', checked=f.strictFiltering, disabled='true') + td + if f.filterMode === 0 + p.no-m-p.nowrap #{__('Do nothing')} + else if f.filterMode === 1 + p.no-m-p.nowrap #{__('Block post')} + else if f.filterMode === 2 + p.no-m-p.nowrap #{__('Ban')} + else + p.no-m-p.nowrap ? + td + p.no-m-p !{f.filterMessage} + td + p.no-m-p(style='text-align:right;') !{f.filterBanDuration} + td.text-center + label + input(type='checkbox', name='filter_ban_appealable', value='true', checked=f.filterBanAppealable, disabled='true') + td + if globalmanage + a.ml-5(href=`/globalmanage/editfilter/${f._id}.html`, style='overflow-wrap: break-word;') [#{__('Edit')}] + else + a.ml-5(href=`/${board._id}/manage/editfilter/${f._id}.html`, style='overflow-wrap: break-word;') [#{__('Edit')}] \ No newline at end of file diff --git a/views/mixins/globalmanagenav.pug b/views/mixins/globalmanagenav.pug index d3c674b0..d849b194 100644 --- a/views/mixins/globalmanagenav.pug +++ b/views/mixins/globalmanagenav.pug @@ -22,6 +22,9 @@ mixin globalmanagenav(selected, upLevel) if permissions.get(Permissions.MANAGE_GLOBAL_NEWS) | a(href=`${upLevel ? '../' : ''}news.html` class=(selected === 'news' ? 'bold' : '')) [#{__('News')}] + if permissions.get(Permissions.MANAGE_GLOBAL_SETTINGS) + | + a(href=`${upLevel ? '../' : ''}filters.html` class=(selected === 'filters' ? 'bold' : '')) [#{__('Filters')}] if permissions.get(Permissions.MANAGE_GLOBAL_SETTINGS) | a(href=`${upLevel ? '../' : ''}settings.html` class=(selected === 'settings' ? 'bold' : '')) [#{__('Settings')}] diff --git a/views/mixins/managenav.pug b/views/mixins/managenav.pug index e81a1b42..445f6d41 100644 --- a/views/mixins/managenav.pug +++ b/views/mixins/managenav.pug @@ -18,6 +18,9 @@ mixin managenav(selected, upLevel) if permissions.get(Permissions.MANAGE_BOARD_LOGS) | a(href=`${upLevel ? '../' : ''}logs.html` class=(selected === 'logs' ? 'bold' : '')) [#{__('Logs')}] + if permissions.get(Permissions.MANAGE_BOARD_SETTINGS) + | + a(href=`${upLevel ? '../' : ''}filters.html` class=(selected === 'filters' ? 'bold' : '')) [#{__('Filters')}] if permissions.get(Permissions.MANAGE_BOARD_SETTINGS) | a(href=`${upLevel ? '../' : ''}settings.html` class=(selected === 'settings' ? 'bold' : '')) [#{__('Settings')}] diff --git a/views/pages/account.pug b/views/pages/account.pug index 1d839cc9..bfef845d 100644 --- a/views/pages/account.pug +++ b/views/pages/account.pug @@ -51,6 +51,9 @@ block content if permissions.get(Permissions.MANAGE_GLOBAL_NEWS) | a(href=`/globalmanage/news.html`) #{__('News')} + if permissions.get(Permissions.MANAGE_GLOBAL_SETTINGS) + | + a(href=`/globalmanage/filters.html`) #{__('Filters')} if permissions.get(Permissions.MANAGE_GLOBAL_SETTINGS) | a(href=`/globalmanage/settings.html`) #{__('Global Settings')} @@ -77,6 +80,8 @@ block content | a(href=`/${b}/manage/logs.html`) #{__('Logs')} | + a(href=`/${b}/manage/filters.html`) #{__('Filters')} + | a(href=`/${b}/manage/settings.html`) #{__('Settings')} | a(href=`/${b}/manage/assets.html`) #{__('Assets')} diff --git a/views/pages/editfilter.pug b/views/pages/editfilter.pug new file mode 100644 index 00000000..9412db80 --- /dev/null +++ b/views/pages/editfilter.pug @@ -0,0 +1,18 @@ +extends ../layout.pug +include ../mixins/managenav.pug +include ../mixins/boardheader.pug + +block head + title /#{board._id}/ - #{__('Edit Filter')} + +block content + +boardheader(__('Edit Filter')) + br + +managenav('filters') + hr(size=1) + h4.mv-5 + a(href='/faq.html#filters') #{__('Filters FAQ')}: + h4.mv-5 #{__('Edit Filter')}: + .form-wrapper.flex-center.mv-10 + form.form-post(action=`/forms/board/${board._id}/editfilter` method='POST') + include ../includes/filtereditform.pug diff --git a/views/pages/globaleditfilter.pug b/views/pages/globaleditfilter.pug new file mode 100644 index 00000000..a751743a --- /dev/null +++ b/views/pages/globaleditfilter.pug @@ -0,0 +1,18 @@ +extends ../layout.pug +include ../mixins/globalmanagenav.pug + +block head + title #{__('Edit Filter')} + +block content + h1.board-title #{__('Global Management')} + br + +globalmanagenav('filters', true) + hr(size=1) + h4.mv-5 + a(href='/faq.html#filters') #{__('Filters FAQ')}: + h4.no-m-p #{__('Edit Filter')}: + include ../includes/stickynav.pug + .form-wrapper.flex-center.mv-10 + form.form-post(action='/forms/global/editfilter' method='POST') + include ../includes/filtereditform.pug \ No newline at end of file diff --git a/views/pages/globalmanagefilters.pug b/views/pages/globalmanagefilters.pug new file mode 100644 index 00000000..a5b5ecd4 --- /dev/null +++ b/views/pages/globalmanagefilters.pug @@ -0,0 +1,37 @@ +extends ../layout.pug +include ../mixins/filter.pug +include ../mixins/globalmanagenav.pug + +block head + title #{__('Filters')} + +block content + h1.board-title #{__('Global Management')} + br + +globalmanagenav('filters') + hr(size=1) + h4.mv-5 + a(href='/faq.html#filters') #{__('Filters FAQ')}: + h4.no-m-p #{__('Add Filter')}: + .form-wrapper.flexleft + form.form-post(action='/forms/global/addfilter', enctype='application/x-www-form-urlencoded', method='POST') + include ../includes/filternewform.pug + if filters.length > 0 + hr(size=1) + h4.no-m-p #{__('Manage Filters')}: + .form-wrapper.flexleft + form.form-post(action='/forms/global/deletefilter', enctype='application/x-www-form-urlencoded', method='POST') + input(type='hidden' name='_csrf' value=csrf) + table + tr + th + th #{__('Filters')} + th #{__('Strict Filtering')} + th #{__('Filter Mode')} + th #{__('Block/Ban Message')} + th #{__('Filter Auto Ban Duration')} + th #{__('Filter Bans Appealable')} + th #{__('Edit')} + each f in filters + +filter(f, true) + input(type='submit', value=__('Delete')) diff --git a/views/pages/globalmanagesettings.pug b/views/pages/globalmanagesettings.pug index 5d2689f1..16a06355 100644 --- a/views/pages/globalmanagesettings.pug +++ b/views/pages/globalmanagesettings.pug @@ -419,28 +419,6 @@ block content .row .label #{__('Expire After Time')} input(type='text' name='block_bypass_expire_after_time' placeholder=__('e.g. 1d') value=settings.blockBypass.expireAfterTime) - .row - h4.mv-5 #{__('Filters')} - .row - .label #{__('Filters')} - textarea(name='filters' placeholder=__('Newline separated')) #{settings.filters.join('\n')} - .row - .label #{__('Strict Filtering')} - label.postform-style.ph-5 - input(type='checkbox', name='strict_filtering', value='true', checked=settings.strictFiltering) - .row - .label #{__('Filter Bans Appealable')} - label.postform-style.ph-5 - input(type='checkbox', name='filter_ban_appealable', value='true', checked=settings.filterBanAppealable) - .row - .label #{__('Filter Mode')} - select(name='filter_mode') - option(value='0', selected=settings.filterMode === 0) #{__('Do nothing')} - option(value='1', selected=settings.filterMode === 1) #{__('Block post')} - option(value='2', selected=settings.filterMode === 2) #{__('Ban')} - .row - .label #{__('Filter Auto Ban Duration')} - input(type='text' name='ban_duration' placeholder=__('e.g. 1w') value=settings.filterBanDuration) .row h4.mv-5 #{__('Flood Protection')} .row @@ -867,19 +845,6 @@ block content .label #{__('Disable anonymizer file posting')} label.postform-style.ph-5 input(type='checkbox', name='board_defaults_disable_anonymizer_file_posting', value='true' checked=settings.boardDefaults.disableAnonymizerFilePosting) - .row - .label #{__('Strict Filtering')} - label.postform-style.ph-5 - input(type='checkbox', name='board_defaults_strict_filtering', value='true' checked=settings.boardDefaults.strictFiltering) - .row - .label #{__('Filter Mode')} - select(name='board_defaults_filter_mode') - option(value='0', selected=settings.boardDefaults.filterMode === 0) #{__('Do nothing')} - option(value='1', selected=settings.boardDefaults.filterMode === 1) #{__('Block post')} - option(value='2', selected=settings.boardDefaults.filterMode === 2) #{__('Ban')} - .row - .label #{__('Filter Auto Ban Duration')} - input(type='text' name='board_defaults_filter_ban_duration' placeholder=__('e.g. 1w') value=settings.boardDefaults.filterBanDuration) .row .label #{__('Allow Video Files')} label.postform-style.ph-5 diff --git a/views/pages/managefilters.pug b/views/pages/managefilters.pug new file mode 100644 index 00000000..83b06c71 --- /dev/null +++ b/views/pages/managefilters.pug @@ -0,0 +1,38 @@ +extends ../layout.pug +include ../mixins/filter.pug +include ../mixins/managenav.pug +include ../mixins/boardheader.pug + +block head + title /#{board._id}/ - #{__('Filters')} + +block content + +boardheader(__('Filters')) + br + +managenav('filters') + hr(size=1) + h4.mv-5 + a(href='/faq.html#filters') #{__('Filters FAQ')}: + h4.mv-5 #{__('Add Filter')}: + .form-wrapper.flexleft + form.form-post(action=`/forms/board/${board._id}/addfilter`, enctype='application/x-www-form-urlencoded', method='POST') + include ../includes/filternewform.pug + if filters.length > 0 + hr(size=1) + h4.no-m-p #{__('Manage Filters')}: + .form-wrapper.flexleft + form.form-post(action=`/forms/board/${board._id}/deletefilter`, enctype='application/x-www-form-urlencoded', method='POST') + input(type='hidden' name='_csrf' value=csrf) + table + tr + th + th #{__('Filters')} + th #{__('Strict Filtering')} + th #{__('Filter Mode')} + th #{__('Block/Ban Message')} + th #{__('Filter Auto Ban Duration')} + th #{__('Filter Bans Appealable')} + th #{__('Edit')} + each f in filters + +filter(f) + input(type='submit', value=__('Delete')) diff --git a/views/pages/managesettings.pug b/views/pages/managesettings.pug index 890be0dc..11334f3f 100644 --- a/views/pages/managesettings.pug +++ b/views/pages/managesettings.pug @@ -310,21 +310,5 @@ block content .row .label #{__('Blocked Countries')} include ../includes/2charisocountries.pug - .row - .label #{__('Filters')} - textarea(name='filters' placeholder=__('Newline separated')) #{board.settings.filters.join('\n')} - .row - .label #{__('Strict Filtering')} - label.postform-style.ph-5 - input(type='checkbox', name='strict_filtering', value='true' checked=board.settings.strictFiltering) - .row - .label #{__('Filter Mode')} - select(name='filter_mode') - option(value='0', selected=board.settings.filterMode === 0) #{__('Do nothing')} - option(value='1', selected=board.settings.filterMode === 1) #{__('Block post')} - option(value='2', selected=board.settings.filterMode === 2) #{__('Ban')} - .row - .label #{__('Filter Auto Ban Duration')} - input(type='text' name='ban_duration' placeholder=__('e.g. 1w') value=board.settings.filterBanDuration) input.row(type='submit', value=__('Save settings'))