Merge branch 'develop' into feature/531-web3-signing

merge-requests/341/head
Thomas Lynch 9 months ago
commit 394c93b140
  1. 9
      CHANGELOG.md
  2. 2
      configs/nginx/snippets/jschan_common_routes.conf
  3. 2
      configs/template.js.example
  4. 2
      controllers/forms/addfilter.js
  5. 7
      db/bans.js
  6. 54
      db/filters.js
  7. 4
      gulp/res/css/themes/tomorrow.css
  8. 28
      gulp/res/css/themes/tomorrow2.css
  9. 4
      gulp/res/css/themes/yotsuba.css
  10. 3
      gulp/res/js/expand.js
  11. 11
      gulp/res/js/forms.js
  12. 2
      gulpfile.js
  13. 3
      lib/middleware/file/filemiddlewares.js
  14. 3
      lib/post/checkfilters.js
  15. 2
      lib/post/filteractions.js
  16. 2
      locales/pt-BR.json
  17. 2
      locales/pt-PT.json
  18. 15
      migrations/1.2.2.js
  19. 1
      models/forms/changeglobalsettings.js
  20. 3
      models/forms/deletepostsfiles.js
  21. 19
      package.json
  22. 2
      test/actions.js
  23. 64
      test/board.js
  24. 53
      test/global.js
  25. 2
      test/integration.test.js
  26. 3
      test/setup.js
  27. 4
      views/pages/globalmanagesettings.pug

@ -1,3 +1,12 @@
### 1.2.2
- Add a global setting to try and URI decode filenames
- Minor pt-PT and pt-BR language fixes
- Remove some unused dependencies
- Npm audit
### 1.2.1
- Bugfix ban upgrades not applying correctly due to bans format change.
### 1.2.0
- Massive filter system overhaul, huge thanks to @disco.
- Filters are no longer just a text box with some options that apply to all filters.

@ -29,7 +29,7 @@ location /captcha {
}
# authed, no cache pages
location ~* ^/((\w+/manage/.*|globalmanage/(reports|bans|recent|boards|globallogs|news|editnews/.*|accounts|editaccount/.*|roles|editrole/.*|settings))|sessions|mypermissions|account|create|twofactor|csrf)\.(html|json)$ {
location ~* ^/((\w+/manage/.*|globalmanage/(reports|bans|recent|boards|filters|globallogs|news|editnews/.*|accounts|editaccount/.*|roles|editrole/.*|settings))|sessions|mypermissions|account|create|twofactor|csrf)\.(html|json)$ {
expires 0;
try_files /dev/null @backend-private;
}

@ -207,6 +207,8 @@ module.exports = {
(spaces dont belong in filenames) */
spaceFileNameReplacement: '_',
uriDecodeFileNames: false,
//options for code block highlighting in posts
highlightOptions: {

@ -27,7 +27,7 @@ module.exports = {
if (errors.length > 0) {
return dynamicResponse(req, res, 400, 'message', {
'title': __('Bad request'),
'errors': req.params.board,
'errors': errors,
'redirect': req.params.board ? `/${req.params.board}/manage/filters.html` : '/globalmanage/filters.html'
});
}

@ -41,10 +41,12 @@ module.exports = {
},
'board': board,
//bypass or pruned IP bans aren't upgraded, duh!
'type': 0,
'ip.type': {
'$lt': 2,
},
//dont allow half -> quarter
'range': {
'$lt': upgradeType
'$lt': upgradeType,
}
}
}, {
@ -59,6 +61,7 @@ module.exports = {
'$substr': substrProjection,
},
'ip.raw': '$ip.raw',
'ip.type': '$ip.type',
}
}, {
'$merge': {

@ -2,6 +2,7 @@
'use strict';
const Mongo = require(__dirname+'/db.js')
, cache = require(__dirname+'/../lib/redis/redis.js')
, db = Mongo.db.collection('filters');
module.exports = {
@ -9,27 +10,40 @@ module.exports = {
db,
// null board retrieves global filters only
findForBoard: (board=null, limit=0) => {
return db.find({'board': board}).sort({
'_id': -1
})
.limit(limit)
.toArray();
findForBoard: async (board=null, limit=0) => {
let filters = await cache.get(`filters:${board}`);
if (filters) {
return filters === 'no_exist' ? [] : filters;
} else {
filters = await db.find({
'board': board
}).sort({
'_id': -1
})
.limit(limit)
.toArray();
if (filters) {
cache.set(`filters:${board}`, filters, 3600);
} else {
cache.set(`filters:${board}`, 'no_exist', 600);
}
}
return filters;
},
count: (board) => {
count: (board=null) => {
return db.countDocuments({'board': board});
},
findOne: (board, id) => {
findOne: (board=null, id) => {
return db.findOne({
'_id': id,
'board': board,
});
},
updateOne: (board, id, filters, strictFiltering, filterMode, filterMessage, filterBanDuration, filterBanAppealable) => {
return db.updateOne({
updateOne: async (board=null, id, filters, strictFiltering, filterMode, filterMessage, filterBanDuration, filterBanAppealable) => {
const updatedFilter = await db.updateOne({
'_id': id,
'board': board,
}, {
@ -42,23 +56,31 @@ module.exports = {
'filterBanAppealable': filterBanAppealable,
}
});
await cache.del(`filters:${board}`);
return updatedFilter;
},
insertOne: (filter) => {
return db.insertOne(filter);
insertOne: async (filter) => {
const insertedFilter = await db.insertOne(filter);
await cache.del(`filters:${filter.board}`);
return insertedFilter;
},
deleteMany: (board, ids) => {
return db.deleteMany({
deleteMany: async (board=null, ids) => {
const deletedFilter = await db.deleteMany({
'_id': {
'$in': ids
},
'board': board
});
await cache.del(`filters:${board}`);
return deletedFilter;
},
deleteBoard: (board) => {
return db.deleteMany({'board': board});
deleteBoard: async (board=null) => {
const deletedFilters = await db.deleteMany({ 'board': board });
await cache.del(`filters:${board}`);
return deletedFilters;
},
deleteAll: () => {

@ -41,6 +41,10 @@ table {
filter: invert(90%);
}
.box-wrap, .tabs li a {
border: 1px solid var(--box-border-color);
}
@media only screen and (max-width: 600px) {
.post-info{

@ -32,21 +32,29 @@
--pinktext-color:#E0727F;
}
.anchor:target + .post-container,
.post-container.highlighted {
.anchor:target + .post-container, .post-container.highlighted {
border: 1px solid var(--highlighted-post-outline-color) !important;
}
.post-container, #float .post-container, .stickynav, .pages, #action-menu, .catalog-tile, #livetext, #threadstats, .collapse, .bottom-reply {
border-bottom: 1px solid var(--post-outline-color);
border-right: 1px solid var(--post-outline-color);
border-top: 1px solid var(--post-color);
border-left: 1px solid var(--post-color);
#float .post-container, #livetext, #threadstats, .navbar, .catalog-tile, .pages, .post-container, .stickynav, .collapse, .bottom-reply {
border-width: 1px!important;
border-color: var(--box-border-color)!important;
}
.navbar{
box-shadow: 0 0 3px 0px #00000030;
}
.post-container.op {
border-color: transparent!important;
}
table {
border: none;
border-spacing: 2px;
border-spacing: 0px;
}
tr:nth-child(2n+1) {
background: unset;
}
.captcha {
@ -54,7 +62,7 @@ table {
}
.box-wrap, .tabs li a {
border: 1px solid #000;
border: 1px solid var(--box-border-color);
}
@media only screen and (max-width: 600px) {

@ -10,13 +10,13 @@
--post-outline-color: #d9bfb7;
--label-color: #fca;
--box-border-color: #000;
--darken: #00000010;
--darken: #ead6ca;
--highlighted-post-color: #f0c0b0;
--highlighted-post-outline-color: #d99f91;
--board-title: #af0a0f;
--hr: #D9BFB7;
--font-color: #800000;
--name-color: #800000;
--name-color: #117743;
--capcode-color: #f00;
--subject-color: #f00;
--link-color: #800;

@ -108,7 +108,8 @@ window.addEventListener('DOMContentLoaded', () => {
}
thumbElement.style.opacity = '0.5';
thumbElement.style.cursor = 'wait';
if (localStorage.getItem('imageloadingbars') == 'true') {
if (localStorage.getItem('imageloadingbars') == 'true'
&& window.URL.createObjectURL) {
const request = new XMLHttpRequest();
request.onprogress = (e) => {
const progress = Math.floor((e.loaded/e.total)*100);

@ -181,14 +181,15 @@ class postFormHandler {
onCancel: () => {},
onDone: () => {
const now = Date.now();
//add replay file if box was checked
let replayBlob;
if (saveReplay) {
const blob = Tegaki.replayRecorder.toBlob();
this.addFile(new File([blob], `${now}-tegaki.tgkr`, { type: 'tegaki/replay' }), { stripFilenames: false });
replayBlob = Tegaki.replayRecorder.toBlob();
}
//add tegaki image
Tegaki.flatten().toBlob(b => {
this.addFile(new File([b], `${now}-tegaki.png`, { type: 'image/png' }), { stripFilenames: false });
Tegaki.flatten().toBlob(imageBlob => {
this.addFile(new File([imageBlob], `${now}-tegaki.png`, { type: 'image/png' }), { stripFilenames: false });
//add replay file
replayBlob && this.addFile(new File([replayBlob], `${now}-tegaki.tgkr`, { type: 'tegaki/replay' }), { stripFilenames: false });
}, 'image/png');
//update file list
this.updateFilesText();

@ -202,10 +202,12 @@ async function wipe() {
await Posts.db.dropIndexes();
await Modlogs.db.dropIndexes();
await CustomPages.db.dropIndexes();
await Filters.db.dropIndexes();
await CustomPages.db.createIndex({ 'board': 1, 'page': 1 }, { unique: true });
await Roles.db.createIndex({ 'permissions': 1 }, { unique: true });
await Modlogs.db.createIndex({ 'board': 1 });
await Files.db.createIndex({ 'count': 1 });
await Filters.db.createIndex({ 'board': 1 });
await Bans.db.createIndex({ 'ip.cloak': 1 , 'board': 1 });
await Bans.db.createIndex({ 'expireAt': 1 }, { expireAfterSeconds: 0 }); //custom expiry, i.e. it will expire when current date > than this date
await Bypass.db.createIndex({ 'expireAt': 1 }, { expireAfterSeconds: 0 });

@ -22,7 +22,7 @@ const { debugLogs } = require(__dirname+'/../../../configs/secrets.js')
});
}
, updateHandlers = () => {
const { globalLimits, filterFileNames, spaceFileNameReplacement } = require(__dirname+'/../../misc/config.js').get;
const { globalLimits, filterFileNames, spaceFileNameReplacement, uriDecodeFileNames } = require(__dirname+'/../../misc/config.js').get;
['flag', 'banner', 'asset', 'post'].forEach(fileType => {
const fileSizeLimit = globalLimits[`${fileType}FilesSize`];
const fileNumLimit = globalLimits[`${fileType}Files`];
@ -45,6 +45,7 @@ const { debugLogs } = require(__dirname+'/../../../configs/secrets.js')
safeFileNames: filterFileNames,
spaceFileNameReplacement,
preserveExtension: 4,
uriDecodeFileNames,
limits: {
totalSize: fileSizeLimit.max,
fileSize: fileSizeLimit.max,

@ -3,8 +3,9 @@
module.exports = (filters, combinedString, strictCombinedString) => {
for (const filter of filters) {
if (filter.filterMode === 0) { continue; } //skip "Do nothing" mode filters
const string = filter.strictFiltering ? strictCombinedString : combinedString;
const hitFilter = filter.filters.find(match => { return string.includes(match.toLowerCase()); });
const hitFilter = filter.filters.find(match => string.includes(match.toLowerCase()) );
if (hitFilter) {
return [ hitFilter, filter.filterMode, filter.filterMessage, filter.filterBanDuration, filter.filterBanAppealable ];
}

@ -10,7 +10,7 @@ module.exports = async (req, res, globalFilter, hitFilter, filterMode,
const { __ } = res.locals;
const blockMessage = __('Your post was blocked by a word filter') + (filterMessage ? ': ' + filterMessage : '');
if (filterMode === 1) {
return dynamicResponse(req, res, 400, 'message', {
'title': __('Bad request'),

@ -51,7 +51,7 @@
"other": "%s meses desde agora"
},
"%s replies": {
"one": "%s reposta",
"one": "%s resposta",
"other": "%s respostas"
},
"%s UIDs": {

@ -51,7 +51,7 @@
"other": "%s meses desde agora"
},
"%s replies": {
"one": "%s reposta",
"one": "%s resposta",
"other": "%s respostas"
},
"%s UIDs": {

@ -0,0 +1,15 @@
'use strict';
module.exports = async(db, redis) => {
console.log('Updating globalsettings to add uriDecodeFileNames');
await db.collection('globalsettings').updateOne({ _id: 'globalsettings' }, {
'$set': {
'uriDecodeFileNames': false,
},
});
console.log('Clearing globalsettings cache');
await redis.deletePattern('globalsettings');
};

@ -197,6 +197,7 @@ module.exports = async (req, res) => {
maxRecentNews: numberSetting(req.body.max_recent_news, oldSettings.maxRecentNews),
filterFileNames: booleanSetting(req.body.filter_file_names, oldSettings.filterFileNames),
spaceFileNameReplacement: req.body.space_file_name_replacement,
uriDecodeFileNames: booleanSetting(req.body.uri_decode_file_names, oldSettings.uriDecodeFileNames),
globalLimits: {
customCss: {
enabled: booleanSetting(req.body.global_limits_custom_css_enabled, oldSettings.globalLimits.customCss.enabled),

@ -19,7 +19,8 @@ module.exports = async (locals, unlinkOnly) => {
return {
filename: file.filename,
hash: file.hash,
thumbextension: file.thumbextension
thumbextension: file.thumbextension,
hasThumb: file.hasThumb,
};
}));
}

@ -1,11 +1,11 @@
{
"name": "jschan",
"version": "1.2.0",
"migrateVersion": "1.2.0",
"version": "1.2.2",
"migrateVersion": "1.2.2",
"description": "",
"main": "server.js",
"dependencies": {
"@fatchan/express-fileupload": "^1.4.2",
"@fatchan/express-fileupload": "^1.4.3",
"@fatchan/gm": "^1.3.2",
"@socket.io/redis-adapter": "^7.2.0",
"bcrypt": "^5.1.0",
@ -22,7 +22,6 @@
"file-type": "^16.5.4",
"fluent-ffmpeg": "^2.1.2",
"form-data": "^4.0.0",
"fs": "0.0.1-security",
"fs-extra": "^10.1.0",
"gulp": "^4.0.2",
"gulp-clean-css": "^4.3.0",
@ -40,18 +39,16 @@
"ioredis": "^4.28.5",
"ip6addr": "^0.2.5",
"mongodb": "^4.16.0",
"node-fetch": "^2.6.11",
"otpauth": "^9.1.2",
"path": "^0.12.7",
"node-fetch": "^2.6.12",
"otpauth": "^9.1.3",
"pm2": "^5.3.0",
"pug": "^3.0.2",
"pug-runtime": "^3.0.1",
"qrcode": "^1.5.3",
"redlock": "^4.2.0",
"sanitize-html": "^2.10.0",
"sanitize-html": "^2.11.0",
"saslprep": "^1.0.3",
"semver": "^7.5.2",
"socket.io": "^4.6.2",
"semver": "^7.5.3",
"socket.io": "^4.7.1",
"socks-proxy-agent": "^6.2.1",
"uid-safe": "^2.1.5",
"unix-crypt-td-js": "^1.1.4",

@ -689,7 +689,7 @@ int main() {...}
_csrf: csrfToken,
checkedbans: banId,
option: 'edit_duration',
ban_duration: '3d',
ban_duration: '1s',
});
const response = await fetch('http://localhost/forms/global/editbans', {
headers: {

@ -231,7 +231,7 @@ testing 123`
});
let filterId;
test('add filter post', async () => {
test('add filter to test board', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,
filters: `notgood
@ -252,25 +252,24 @@ bad words`,
redirect: 'manual',
});
expect(response.ok).toBe(true);
const filterPage = await fetch('http://localhost/test/manage/filter.html', {
const filterPage = await fetch('http://localhost/test/manage/filters.html', {
headers: {
'cookie': sessionCookie,
},
}).then(res => res.text());
const checkIndex = filterPage.indexOf('name="checkedfilter" value="');
filterId = filterPage.substring(checkIndex+28, checkIndex+28+24);
const checkIndex = filterPage.indexOf('name="checkedfilters" value="');
filterId = filterPage.substring(checkIndex+29, checkIndex+29+24);
});
test('edit filter post', async () => {
test('edit filter on test board', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,
board: test,
filter_id: filterId,
filters: 'edited filters',
strict_filtering: 'true',
filter_mode: '0',
filter_mode: '1',
filter_message: 'edited message',
filter_ban_duration: '0'
filter_ban_duration: '1s'
// filter_ban_appealable omitted to change to false
});
const response = await fetch('http://localhost/forms/board/test/editfilter', {
@ -283,7 +282,7 @@ bad words`,
redirect: 'manual',
});
expect(response.ok).toBe(true);
const filterPage = await fetch('http://localhost/test/manage/filter.html', {
const filterPage = await fetch('http://localhost/test/manage/filters.html', {
headers: {
'cookie': sessionCookie,
},
@ -292,10 +291,39 @@ bad words`,
expect(editTextIndex).not.toBe(-1);
});
test('delete filter post', async () => {
test('make a post that doesnt hit board filter', async () => {
const params = new URLSearchParams();
params.append('message', 'blahblahblah');
params.append('captcha', '000000');
const response = await fetch('http://localhost/forms/board/test/post', {
headers: {
'x-using-xhr': 'true',
},
method: 'POST',
body: params
});
expect(response.ok).toBe(true);
});
test('make a post that hits board filter', async () => {
const params = new URLSearchParams();
params.append('message', 'edited filters');
params.append('captcha', '000000');
const response = await fetch('http://localhost/forms/board/test/post', {
headers: {
'x-using-xhr': 'true',
},
method: 'POST',
body: params
});
expect(response.ok).not.toBe(true);
await new Promise(res => setTimeout(res, 10000)); //let ban expire
});
test('delete test board filter', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,
checkedfilter: filterId,
checkedfilters: filterId,
});
const response = await fetch('http://localhost/forms/board/test/deletefilter', {
headers: {
@ -309,6 +337,20 @@ bad words`,
expect(response.ok).toBe(true);
});
test('make a post that passes the deleted filter', async () => {
const params = new URLSearchParams();
params.append('message', 'editing filter');
params.append('captcha', '000000');
const response = await fetch('http://localhost/forms/board/test/post', {
headers: {
'x-using-xhr': 'true',
},
method: 'POST',
body: params
});
expect(response.ok).toBe(true);
});
test('add staff', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,

@ -99,7 +99,7 @@ testing 123`
});
let filterId;
test('add filter post', async () => {
test('add global filter', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,
filters: `notgood
@ -120,24 +120,24 @@ bad words`,
redirect: 'manual',
});
expect(response.ok).toBe(true);
const filterPage = await fetch('http://localhost/globalmanage/filter.html', {
const filterPage = await fetch('http://localhost/globalmanage/filters.html', {
headers: {
'cookie': sessionCookie,
},
}).then(res => res.text());
const checkIndex = filterPage.indexOf('name="checkedfilter" value="');
filterId = filterPage.substring(checkIndex+28, checkIndex+28+24);
const checkIndex = filterPage.indexOf('name="checkedfilters" value="');
filterId = filterPage.substring(checkIndex+29, checkIndex+29+24);
});
test('edit filter post', async () => {
test('edit global filter', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,
filter_id: filterId,
filters: 'edited filters',
filters: 'edited globalfilters',
strict_filtering: 'true',
filter_mode: '0',
filter_mode: '1',
filter_message: 'edited message',
filter_ban_duration: '0'
filter_ban_duration: '1s'
// filter_ban_appealable omitted to change to false
});
const response = await fetch('http://localhost/forms/global/editfilter', {
@ -150,19 +150,48 @@ bad words`,
redirect: 'manual',
});
expect(response.ok).toBe(true);
const filterPage = await fetch('http://localhost/globalmanage/filter.html', {
const filterPage = await fetch('http://localhost/globalmanage/filters.html', {
headers: {
'cookie': sessionCookie,
},
}).then(res => res.text());
const editTextIndex = filterPage.indexOf('edited filters');
const editTextIndex = filterPage.indexOf('edited globalfilters');
expect(editTextIndex).not.toBe(-1);
});
test('delete filter post', async () => {
test('make post that doesnt hit global filter', async () => {
const params = new URLSearchParams();
params.append('message', 'blahblahblah');
params.append('captcha', '000000');
const response = await fetch('http://localhost/forms/board/test/post', {
headers: {
'x-using-xhr': 'true',
},
method: 'POST',
body: params
});
expect(response.ok).toBe(true);
});
test('make post that hits global filter', async () => {
const params = new URLSearchParams();
params.append('message', 'edited globalfilters');
params.append('captcha', '000000');
const response = await fetch('http://localhost/forms/board/test/post', {
headers: {
'x-using-xhr': 'true',
},
method: 'POST',
body: params
});
expect(response.ok).not.toBe(true);
await new Promise(res => setTimeout(res, 10000)); //let ban expire
});
test('delete global filter', async () => {
const params = new URLSearchParams({
_csrf: csrfToken,
checkedfilter: filterId,
checkedfilters: filterId,
});
const response = await fetch('http://localhost/forms/global/deletefilter', {
headers: {

@ -2,8 +2,8 @@ describe('run integration tests', () => {
require('./setup.js')();
require('./posting.js')();
require('./global.js')();
require('./actions.js')();
require('./board.js')();
require('./actions.js')();
require('./pages.js')();
require('./cleanup.js')();
require('./twofactor.js')();

@ -103,7 +103,7 @@ module.exports = () => describe('login and create test board', () => {
archive_links: 'https://archive.today/?run=1&url=%s',
reverse_links: 'https://tineye.com/search?url=%s',
prune_modlogs: '30',
default_ban_duration: '31536000000',
default_ban_duration: '1000',
quote_limit: '25',
preview_replies: '5',
sticky_preview_replies: '5',
@ -178,6 +178,7 @@ module.exports = () => describe('login and create test board', () => {
global_limits_custom_css_filters: '@\nurl(',
global_limits_custom_css_strict: 'true',
global_limits_custom_css_max: '10000',
global_limits_filters_max: 100,
global_limits_field_length_name: '100',
global_limits_field_length_email: '100',
global_limits_field_length_subject: '100',

@ -586,6 +586,10 @@ block content
.row
.label #{__('Space File Name Replacement')}
input(type='text', name='space_file_name_replacement', value=settings.spaceFileNameReplacement)
.row
.label #{__('URI Decode File Names')}
label.postform-style.ph-5
input(type='checkbox', name='uri_decode_file_names', value='true' checked=settings.uriDecodeFileNames)
.row
.label #{__('Thumbnail File Extension')}
input(type='text' name='thumb_extension' value=settings.thumbExtension)

Loading…
Cancel
Save