Merge branch 'new-dev' into 'master'

New dev

Closes #316 and #319

See merge request fatchan/jschan!212
merge-requests/218/head
Thomas Lynch 4 years ago
commit 9351a4bd1f
  1. 21
      configs/main.js.example
  2. 2
      configs/nginx/nginx.example
  3. 2
      configs/nginx/nginx_no_https.example
  4. 8
      controllers/forms/makepost.js
  5. 4
      db/posts.js
  6. 2
      gulp/res/css/style.css
  7. BIN
      gulp/res/img/attachment.png
  8. BIN
      gulp/res/img/audio.png
  9. 17
      gulp/res/js/hideimages.js
  10. 16
      gulp/res/js/hover.js
  11. 51
      gulpfile.js
  12. 4
      helpers/captcha/verify.js
  13. 6
      helpers/checks/blockbypass.js
  14. 2
      helpers/checks/dnsbl.js
  15. 8
      helpers/checks/torprebypass.js
  16. 10
      helpers/countries.js
  17. 4
      helpers/filemiddlewares.js
  18. 6
      helpers/geoip.js
  19. 2
      helpers/processip.js
  20. 6
      helpers/tasks.js
  21. 2
      migrations/index.js
  22. 2
      migrations/migration-0.0.17.js
  23. 12
      migrations/migration-0.0.18.js
  24. 11
      migrations/migration-0.0.19.js
  25. 2
      models/forms/changeboardsettings.js
  26. 2
      models/forms/makepost.js
  27. 2
      models/pages/captcha.js
  28. 8
      package-lock.json
  29. 2
      package.json
  30. 7
      schedules/index.js
  31. 53
      schedules/ips.js
  32. 4
      socketio.js
  33. 2
      views/custompages/faq.pug.example
  34. 4
      views/pages/managesettings.pug

@ -71,11 +71,12 @@ module.exports = {
cacheTime: 3600 //in seconds, idk whats a good value cacheTime: 3600 //in seconds, idk whats a good value
}, },
//disable file posting over .onion globally, overrides any board setting. //disable file posting over anonymizers globally, overrides any board setting.
disableOnionFilePosting: false, disableAnonymizerFilePosting: false,
//count .onion posters as "users" in stats. if set to false, all .onion is counted as a single user. doesnt affect pph stat. /* count "IP"s (bypass ids) for anonymizers as "users" in stats. if set to false, anonymous users are counted as a single user. doesnt affect pph stat.
statsCountOnionUsers: true, you can use this setting to prevent spam over anonymizers from inflating user stats */
statsCountAnonymizers: true,
floodTimers: { //basic delays to stop flooding, in ms. 0 to disable floodTimers: { //basic delays to stop flooding, in ms. 0 to disable
sameContentSameIp: 120000, //same message or any file from same ip sameContentSameIp: 120000, //same message or any file from same ip
@ -86,7 +87,7 @@ module.exports = {
//block bypasses //block bypasses
blockBypass: { blockBypass: {
enabled: false, enabled: false,
forceOnion: true, //option to override blockbypass setting for .onion users forceAnonymizers: true, //option to override blockbypass setting for .onion users
expireAfterUses: 40, //however many (attempted) posts per block bypass captcha expireAfterUses: 40, //however many (attempted) posts per block bypass captcha
expireAfterTime: 86400000, //expiry in ms regardless if the limit was reached, default 1 day expireAfterTime: 86400000, //expiry in ms regardless if the limit was reached, default 1 day
bypassDnsbl: false, bypassDnsbl: false,
@ -122,9 +123,11 @@ module.exports = {
//max wait time in ms for obtaining locks for saving files //max wait time in ms for obtaining locks for saving files
lockWait: 3000, lockWait: 3000,
//optionally prune oldest modlog entries (prunes when newer modlog entries are generated i.e. dead boards wont have older logs pruned) //optionally prune modlog entries older than x days, false to disable (prunes when newer modlog entries are generated i.e. dead boards wont have older logs pruned)
pruneModlogs: true, pruneModlogs: 30,
pruneAfterDays: 30,
//option to prune ips on posts older than x days, false to disable
pruneIps: false,
//enable the webring (also copy configs/webring.json.example -> configs/webring.json and edit) //enable the webring (also copy configs/webring.json.example -> configs/webring.json and edit)
enableWebring: false, enableWebring: false,
@ -374,7 +377,7 @@ module.exports = {
defaultName: 'Anon', defaultName: 'Anon',
customCSS: null, customCSS: null,
blockedCountries: [], //2 char ISO country codes to block blockedCountries: [], //2 char ISO country codes to block
disableOnionFilePosting: false, disableAnonymizerFilePosting: false,
filters: [], //words/phrases to block filters: [], //words/phrases to block
filterMode: 0, //0=nothing, 1=prevent post, 2=auto ban filterMode: 0, //0=nothing, 1=prevent post, 2=auto ban
filterBanDuration: 0, //duration (in ms) to ban if filter mode=2 filterBanDuration: 0, //duration (in ms) to ban if filter mode=2

@ -9,7 +9,7 @@ server {
server_tokens off; server_tokens off;
add_header Cache-Control "public"; add_header Cache-Control "public";
add_header Content-Security-Policy "default-src 'self'; img-src 'self' blob:; object-src 'self' blob:; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube.com/embed/ https://www.bitchute.com/embed/; connect-src 'self' wss://doimain.com"; add_header Content-Security-Policy "default-src 'self'; img-src 'self' blob:; object-src 'self' blob:; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube.com/embed/ https://www.bitchute.com/embed/; connect-src 'self' wss://domain.com/";
add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin" always; add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin" always;
add_header X-Frame-Options "sameorigin" always; add_header X-Frame-Options "sameorigin" always;
add_header X-Content-Type-Options "nosniff" always; add_header X-Content-Type-Options "nosniff" always;

@ -305,7 +305,7 @@ server {
# location ~* \.json$ { # location ~* \.json$ {
# expires 0; # expires 0;
# root /path/to/jschan/static/json; # root /path/to/jschan/static/json;
# try_files $uri =404; # try_files $uri @backend;
# #json doesnt hit backend if it doesnt exist yet. # #json doesnt hit backend if it doesnt exist yet.
# } # }
# #

@ -4,7 +4,7 @@ const makePost = require(__dirname+'/../../models/forms/makepost.js')
, deleteTempFiles = require(__dirname+'/../../helpers/files/deletetempfiles.js') , deleteTempFiles = require(__dirname+'/../../helpers/files/deletetempfiles.js')
, dynamicResponse = require(__dirname+'/../../helpers/dynamic.js') , dynamicResponse = require(__dirname+'/../../helpers/dynamic.js')
, pruneFiles = require(__dirname+'/../../schedules/prune.js') , pruneFiles = require(__dirname+'/../../schedules/prune.js')
, { pruneImmediately, globalLimits, disableOnionFilePosting } = require(__dirname+'/../../configs/main.js') , { pruneImmediately, globalLimits, disableAnonymizerFilePosting } = require(__dirname+'/../../configs/main.js')
, { Files } = require(__dirname+'/../../db/'); , { Files } = require(__dirname+'/../../db/');
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
@ -15,10 +15,10 @@ module.exports = async (req, res, next) => {
if ((!req.body.message || res.locals.messageLength === 0) && res.locals.numFiles === 0) { if ((!req.body.message || res.locals.messageLength === 0) && res.locals.numFiles === 0) {
errors.push('Posts must include a message or file'); errors.push('Posts must include a message or file');
} }
if (res.locals.tor if (res.locals.anonymizer
&& (disableOnionFilePosting || res.locals.board.settings.disableOnionFilePosting) && (disableAnonymizerFilePosting || res.locals.board.settings.disableAnonymizerFilePosting)
&& res.locals.numFiles > 0) { && res.locals.numFiles > 0) {
errors.push(`Posting files through the .onion address has been disabled ${disableOnionFilePosting ? 'globally' : 'on this board'}`); errors.push(`Posting files through anonymizers has been disabled ${disableAnonymizerFilePosting ? 'globally' : 'on this board'}`);
} }
if (res.locals.numFiles > res.locals.board.settings.maxFiles) { if (res.locals.numFiles > res.locals.board.settings.maxFiles) {
errors.push(`Too many files. Max files per post ${res.locals.board.settings.maxFiles < globalLimits.postFiles.max ? 'on this board ' : ''}is ${res.locals.board.settings.maxFiles}`); errors.push(`Too many files. Max files per post ${res.locals.board.settings.maxFiles < globalLimits.postFiles.max ? 'on this board ' : ''}is ${res.locals.board.settings.maxFiles}`);

@ -4,7 +4,7 @@ const Mongo = require(__dirname+'/db.js')
, Boards = require(__dirname+'/boards.js') , Boards = require(__dirname+'/boards.js')
, Stats = require(__dirname+'/stats.js') , Stats = require(__dirname+'/stats.js')
, db = Mongo.db.collection('posts') , db = Mongo.db.collection('posts')
, { quoteLimit, previewReplies, stickyPreviewReplies, statsCountOnionUsers, , { quoteLimit, previewReplies, stickyPreviewReplies, statsCountAnonymizers,
ipHashPermLevel, early404Replies, early404Fraction } = require(__dirname+'/../configs/main.js'); ipHashPermLevel, early404Replies, early404Fraction } = require(__dirname+'/../configs/main.js');
module.exports = { module.exports = {
@ -474,7 +474,7 @@ module.exports = {
//insert the post itself //insert the post itself
const postMongoId = await db.insertOne(data).then(result => result.insertedId); //_id of post const postMongoId = await db.insertOne(data).then(result => result.insertedId); //_id of post
const statsIp = (statsCountOnionUsers === false && res.locals.tor === true) ? null : data.ip.single; const statsIp = (statsCountAnonymizers === false && res.locals.anonymizer === true) ? null : data.ip.single;
await Stats.updateOne(board._id, statsIp, data.thread == null); await Stats.updateOne(board._id, statsIp, data.thread == null);
//add backlinks to the posts this post quotes //add backlinks to the posts this post quotes

@ -501,7 +501,7 @@ th {
} }
.fw td, .fw th { .fw td, .fw th {
width: 15%; /*Fixes log tables when large actions are taken*/ width: 8%; /*Fixes log tables when large actions are taken*/
} }
td, th { td, th {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.8 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

@ -4,7 +4,7 @@ let imageSourcesList;
const toggleAllHidden = (state) => imageSources.forEach(i => toggleSource(i, state)); const toggleAllHidden = (state) => imageSources.forEach(i => toggleSource(i, state));
const toggleSource = (source, state) => { const toggleSource = (source, state) => {
const images = document.querySelectorAll(`img.file-thumb[src="${source}"]`); const images = document.querySelectorAll(`img.file-thumb[src="${source}"], img.catalog-thumb[src="${source}"]`);
images.forEach(i => i.classList[state?'add':'remove']('vh')); images.forEach(i => i.classList[state?'add':'remove']('vh'));
} }
@ -26,13 +26,22 @@ document.querySelectorAll('.hide-image').forEach(el => {
const handleHiddenImages = (e) => { const handleHiddenImages = (e) => {
//hide any images from this post that should already be hidden //hide any images from this post that should already be hidden
const hasHiddenImages = e.detail.json.files.forEach(f => { const hasHiddenImages = e.detail.json.files.forEach(f => {
if (imageSources.has(f.filename)) { let hideFilename = '/file/';
toggleSource(f.filename, true); if (f.hasThumb) {
hideFilename += `thumb-${f.hash}${f.thumbextension}`
} else {
hideFilename += f.filename;
}
if (imageSources.has(hideFilename)) {
toggleSource(hideFilename, true);
} }
}); });
//add the hide toggle link and event listener //add the hide toggle link and event listener
if (!e.detail.hover) { if (!e.detail.hover) {
e.detail.post.querySelector('.hide-image').addEventListener('click', toggleHandler, false); const hideButtons = e.detail.post.querySelectorAll('.hide-image');
for (let i = 0; i < hideButtons.length; i++) {
hideButtons[i].addEventListener('click', toggleHandler, false);
}
} }
} }

@ -59,6 +59,7 @@ window.addEventListener('DOMContentLoaded', (event) => {
clone.appendChild(post.cloneNode(true)); clone.appendChild(post.cloneNode(true));
document.body.appendChild(clone); document.body.appendChild(clone);
setFloatPos(quote, clone, xpos, ypos); setFloatPos(quote, clone, xpos, ypos);
return clone;
}; };
const toggleHighlightPost = async function (e) { const toggleHighlightPost = async function (e) {
@ -85,13 +86,12 @@ window.addEventListener('DOMContentLoaded', (event) => {
lastHover = loading; lastHover = loading;
const hash = this.hash.substring(1); const hash = this.hash.substring(1);
const anchor = document.getElementById(hash); const anchor = document.getElementById(hash);
let hoveredPost; let hoveredPost, postJson;
if (anchor if (anchor
&& jsonPath.split('/')[1] === anchor.nextSibling.dataset.board) { && jsonPath.split('/')[1] === anchor.nextSibling.dataset.board) {
hoveredPost = anchor.nextSibling; hoveredPost = anchor.nextSibling;
} else { } else {
let hovercache = localStorage.getItem(`hovercache-${jsonPath}`); let hovercache = localStorage.getItem(`hovercache-${jsonPath}`);
let postJson;
if (hovercache) { if (hovercache) {
hovercache = JSON.parse(hovercache); hovercache = JSON.parse(hovercache);
if (hovercache.postId == hash) { if (hovercache.postId == hash) {
@ -135,6 +135,13 @@ window.addEventListener('DOMContentLoaded', (event) => {
const wrap = document.createElement('div'); const wrap = document.createElement('div');
wrap.innerHTML = postHtml; wrap.innerHTML = postHtml;
hoveredPost = wrap.firstChild.nextSibling; hoveredPost = wrap.firstChild.nextSibling;
}
if (hovering && !isVisible(hoveredPost)) {
hoveredPost = floatPost(this, hoveredPost, e.clientX, e.clientY);
} else {
hovering ? hoveredPost.classList.add('hoverhighlighted') : hoveredPost.classList.remove('hoverhighlighted');
}
if (postJson) {
//need this event so handlers like post hiding still apply to hover introduced posts //need this event so handlers like post hiding still apply to hover introduced posts
const newPostEvent = new CustomEvent('addPost', { const newPostEvent = new CustomEvent('addPost', {
detail: { detail: {
@ -147,11 +154,6 @@ window.addEventListener('DOMContentLoaded', (event) => {
window.dispatchEvent(newPostEvent); window.dispatchEvent(newPostEvent);
} }
toggleDottedUnderlines(hoveredPost, thisId); toggleDottedUnderlines(hoveredPost, thisId);
if (hovering && !isVisible(hoveredPost)) {
floatPost(this, hoveredPost, e.clientX, e.clientY);
} else {
hovering ? hoveredPost.classList.add('hoverhighlighted') : hoveredPost.classList.remove('hoverhighlighted');
}
} }
for (let i = 0; i < quotes.length; i++) { for (let i = 0; i < quotes.length; i++) {

@ -16,7 +16,7 @@ const gulp = require('gulp')
, pug = require('pug') , pug = require('pug')
, gulppug = require('gulp-pug') , gulppug = require('gulp-pug')
, { migrateVersion } = require(__dirname+'/package.json') , { migrateVersion } = require(__dirname+'/package.json')
, { createHash, randomBytes } = require('crypto') , { randomBytes } = require('crypto')
, paths = { , paths = {
styles: { styles: {
src: 'gulp/res/css/', src: 'gulp/res/css/',
@ -59,54 +59,11 @@ async function password() {
} }
async function ips() { async function ips() {
/*
prune IPs from old posts (actually, rehash them with a temporary random salt to maintain
post history and prevent *-by-ip action unintentionally deleting many posts)
NOTE: ips may still remain in the following collections:
- bans, because bans need the IP to function
- modlog actioner ips, modlogs are already auto-pruned
- ratelimits, these only last 1 minute
- stats, these last max of 24 hours
*/
const Mongo = require(__dirname+'/db/db.js') const Mongo = require(__dirname+'/db/db.js')
await Mongo.connect(); await Mongo.connect();
const Redis = require(__dirname+'/redis.js') const Redis = require(__dirname+'/redis.js')
const { Posts } = require(__dirname+'/db/'); const ipSchedule = require(__dirname+'/schedules/ips.js');
const beforeDate = new Date(); await ipSchedule();
beforeDate.setDate(beforeDate.getDate() - 7); //7 days in the past, static number for now until i implement yargs or similar
const beforeDateMongoId = Mongo.ObjectId.createFromTime(Math.floor(beforeDate.getTime()/1000));
const tempIpHashSecret = randomBytes(20).toString('base64');
const bulkWrites = [];
await Posts.db.find({
_id: {
$lte: beforeDateMongoId,
},
'ip.pruned': {
$ne: true
}
}).forEach(post => {
const randomIP = createHash('sha256').update(tempIpHashSecret + post.ip.single).digest('base64');
bulkWrites.push({
updateOne: {
filter: {
_id: post._id,
},
update: {
$set: {
'ip.pruned': true,
'ip.raw': randomIP,
'ip.single': randomIP,
'ip.qrange': randomIP,
'ip.hrange': randomIP,
}
}
}
});
});
console.log(`Randomising ip on ${bulkWrites.length} posts`);
if (bulkWrites.length.length > 0) {
await Posts.db.bulkWrite(bulkWrites);
}
Redis.redisClient.quit(); Redis.redisClient.quit();
return Mongo.client.close(); return Mongo.client.close();
} }
@ -160,7 +117,7 @@ async function wipe() {
await Posts.db.dropIndexes() await Posts.db.dropIndexes()
await Modlogs.db.dropIndexes() await Modlogs.db.dropIndexes()
await CustomPages.db.dropIndexes() await CustomPages.db.dropIndexes()
await CustomPages.db.createIndex({ 'board': 1, 'url': 1 }, { unique: true }) await CustomPages.db.createIndex({ 'board': 1, 'page': 1 }, { unique: true })
await Modlogs.db.createIndex({ 'board': 1 }) await Modlogs.db.createIndex({ 'board': 1 })
await Files.db.createIndex({ 'count': 1 }) await Files.db.createIndex({ 'count': 1 })
await Bans.db.createIndex({ 'ip.single': 1 , 'board': 1 }) await Bans.db.createIndex({ 'ip.single': 1 , 'board': 1 })

@ -12,7 +12,7 @@ const { Ratelimits } = require(__dirname+'/../../db/')
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
//already solved in pre stage for getting bypassID for "ip" //already solved in pre stage for getting bypassID for "ip"
if (res.locals.tor && res.locals.solvedCaptcha) { if (res.locals.anonymizer && res.locals.solvedCaptcha) {
return next(); return next();
} }
@ -49,7 +49,7 @@ module.exports = async (req, res, next) => {
//for builtin captchas, clear captchaid cookie, delete file and reset quota //for builtin captchas, clear captchaid cookie, delete file and reset quota
res.clearCookie('captchaid'); res.clearCookie('captchaid');
await Promise.all([ await Promise.all([
!res.locals.tor && Ratelimits.resetQuota(res.locals.ip.single, 'captcha'), !res.locals.anonymizer && Ratelimits.resetQuota(res.locals.ip.single, 'captcha'),
remove(`${uploadDirectory}/captcha/${captchaId}.jpg`) remove(`${uploadDirectory}/captcha/${captchaId}.jpg`)
]); ]);
} }

@ -11,8 +11,8 @@ module.exports = async (req, res, next) => {
if (res.locals.preFetchedBypassId //if they already have a bypass if (res.locals.preFetchedBypassId //if they already have a bypass
|| (!blockBypass.enabled //or if block bypass isnt enabled || (!blockBypass.enabled //or if block bypass isnt enabled
&& (!blockBypass.forceOnion //and we dont force it for .onion && (!blockBypass.forceAnonymizers //and we dont force it for anonymizer
|| !res.locals.tor))) { //or they arent a .onion || !res.locals.anonymizer))) { //or they arent on an anonymizer
return next(); return next();
} }
@ -45,7 +45,7 @@ module.exports = async (req, res, next) => {
if (bypass //if they have a valid bypass if (bypass //if they have a valid bypass
&& (bypass.uses < blockBypass.expireAfterUses //and its not overused && (bypass.uses < blockBypass.expireAfterUses //and its not overused
|| (res.locals.tor && !blockBypass.forceOnion))) { //OR its disabled for .onion, which ignores usage check || (res.locals.anonymizer && !blockBypass.forceAnonymizers))) { //OR its forced for anonymizers
return next(); return next();
} }

@ -9,7 +9,7 @@ const cache = require(__dirname+'/../../redis.js')
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
if (dnsbl.enabled && dnsbl.blacklists.length > 0 //if dnsbl enabled and has more than 0 blacklists if (dnsbl.enabled && dnsbl.blacklists.length > 0 //if dnsbl enabled and has more than 0 blacklists
&& !res.locals.tor //tor cant be dnsbl'd && !res.locals.anonymizer //anonymizers cant be dnsbl'd
&& (!res.locals.blockBypass || !blockBypass.bypassDnsbl)) { //and there is no valid block bypass, or they do not bypass dnsbl && (!res.locals.blockBypass || !blockBypass.bypassDnsbl)) { //and there is no valid block bypass, or they do not bypass dnsbl
const ip = req.headers[ipHeader] || req.connection.remoteAddress; const ip = req.headers[ipHeader] || req.connection.remoteAddress;
let isBlacklisted = await cache.get(`blacklisted:${ip}`); let isBlacklisted = await cache.get(`blacklisted:${ip}`);

@ -12,14 +12,14 @@ const { Bypass } = require(__dirname+'/../../db/')
module.exports = async (req, res, next) => { module.exports = async (req, res, next) => {
//early bypass is only needed for tor users //early bypass is only needed for anonymizer users
if (!res.locals.tor) { if (!res.locals.anonymizer) {
return next(); return next();
} }
let bypassId = req.signedCookies.bypassid; let bypassId = req.signedCookies.bypassid;
if (blockBypass.enabled || blockBypass.forceOnion) { if (blockBypass.enabled || blockBypass.forceAnonymizers) {
const input = req.body.captcha; const input = req.body.captcha;
const captchaId = req.cookies.captchaid; const captchaId = req.cookies.captchaid;
if (input && !bypassId) { if (input && !bypassId) {
@ -46,7 +46,7 @@ module.exports = async (req, res, next) => {
if (res.locals.solvedCaptcha //if they just solved a captcha if (res.locals.solvedCaptcha //if they just solved a captcha
|| (!blockBypass.enabled //OR blockbypass isnt enabled || (!blockBypass.enabled //OR blockbypass isnt enabled
&& !blockBypass.forceOnion //AND its not forced for .onion && !blockBypass.forceAnonymizers //AND its not forced for anonymizers
&& !bypassId)) { //AND they dont already have one, && !bypassId)) { //AND they dont already have one,
//then give the user a bypass id //then give the user a bypass id
const newBypass = await Bypass.getBypass(); const newBypass = await Bypass.getBypass();

@ -2,8 +2,11 @@
const countries = require('i18n-iso-countries') const countries = require('i18n-iso-countries')
, countryNamesMap = countries.getNames('en') , countryNamesMap = countries.getNames('en')
, countryCodes = ['EU', 'XX', 'T1', 'TOR', 'LOKI'] , extraCountryCodes = ['EU', 'XX', 'T1']
.concat(Object.keys(countryNamesMap)); , anonymizerCountryCodes = ['TOR', 'LOKI']
, anonymizerCountryCodesSet = new Set(anonymizerCountryCodes)
, countryCodes = Object.keys(countryNamesMap)
.concat(extraCountryCodes, anonymizerCountryCodes);
//this dumb library conveniently includes 2 names for some countries... //this dumb library conveniently includes 2 names for some countries...
Object.entries(countryNamesMap) Object.entries(countryNamesMap)
@ -19,4 +22,7 @@ countryNamesMap['LOKI'] = 'Lokinet SNApp';
module.exports = { module.exports = {
countryNamesMap, countryNamesMap,
countryCodes, countryCodes,
isAnonymizer: (code) => {
return anonymizerCountryCodesSet.has(code);
},
} }

@ -63,14 +63,14 @@ module.exports = {
}), }),
handlePostFilesEarlyTor: (req, res, next) => { handlePostFilesEarlyTor: (req, res, next) => {
if (res.locals.tor) { if (res.locals.anonymizer) {
return postFiles(req, res, next); return postFiles(req, res, next);
} }
return next(); return next();
}, },
handlePostFiles: (req, res, next) => { handlePostFiles: (req, res, next) => {
if (res.locals.tor) { if (res.locals.anonymizer) {
return next(); return next();
} }
return postFiles(req, res, next); return postFiles(req, res, next);

@ -1,11 +1,11 @@
'use strict' 'use strict'
const { countryNamesMap } = require(__dirname+'/countries.js') const { countryNamesMap, isAnonymizer } = require(__dirname+'/countries.js')
, { countryCodeHeader } = require(__dirname+'/../configs/main.js'); , { countryCodeHeader } = require(__dirname+'/../configs/main.js')
module.exports = (req, res, next) => { module.exports = (req, res, next) => {
const code = req.headers[countryCodeHeader] || 'XX'; const code = req.headers[countryCodeHeader] || 'XX';
res.locals.tor = code === 'TOR' || code === 'LOKI'; //NOTE: for ticket #312 change this to use a single x-anonymizer header res.locals.anonymizer = isAnonymizer(code);
res.locals.country = { res.locals.country = {
code, code,
name: countryNamesMap[code], name: countryNamesMap[code],

@ -9,7 +9,7 @@ const { ipHeader, ipHashPermLevel } = require(__dirname+'/../configs/main.js')
module.exports = (req, res, next) => { module.exports = (req, res, next) => {
//tor user ip uses bypass id, if they dont have one send to blockbypass //tor user ip uses bypass id, if they dont have one send to blockbypass
if (res.locals.tor) { if (res.locals.anonymizer) {
const pseudoIp = res.locals.preFetchedBypassId || req.signedCookies.bypassid; const pseudoIp = res.locals.preFetchedBypassId || req.signedCookies.bypassid;
res.locals.ip = { res.locals.ip = {
raw: pseudoIp, raw: pseudoIp,

@ -4,7 +4,7 @@ const Mongo = require(__dirname+'/../db/db.js')
, timeUtils = require(__dirname+'/timeutils.js') , timeUtils = require(__dirname+'/timeutils.js')
, uploadDirectory = require(__dirname+'/files/uploadDirectory.js') , uploadDirectory = require(__dirname+'/files/uploadDirectory.js')
, { remove } = require('fs-extra') , { remove } = require('fs-extra')
, { debugLogs, pruneModlogs, pruneAfterDays, enableWebring, maxRecentNews } = require(__dirname+'/../configs/main.js') , { debugLogs, pruneModlogs, enableWebring, maxRecentNews } = require(__dirname+'/../configs/main.js')
, { CustomPages, Stats, Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/') , { CustomPages, Stats, Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/')
, cache = require(__dirname+'/../redis.js') , cache = require(__dirname+'/../redis.js')
, render = require(__dirname+'/render.js') , render = require(__dirname+'/render.js')
@ -188,9 +188,9 @@ module.exports = {
const label = `/${options.board._id}/logs.html`; const label = `/${options.board._id}/logs.html`;
const start = process.hrtime(); const start = process.hrtime();
let dates = await Modlogs.getDates(options.board); let dates = await Modlogs.getDates(options.board);
if (pruneModlogs === true) { if (pruneModlogs) {
const pruneLogs = []; const pruneLogs = [];
const pruneAfter = new Date(Date.now()-timeUtils.DAY*pruneAfterDays); const pruneAfter = new Date(Date.now()-timeUtils.DAY*pruneModlogs);
dates = dates.filter(date => { dates = dates.filter(date => {
const { year, month, day } = date.date; const { year, month, day } = date.date;
if (new Date(year, month-1, day) > pruneAfter) { //-1 for 0-index months if (new Date(year, month-1, day) > pruneAfter) { //-1 for 0-index months

@ -18,4 +18,6 @@ module.exports = {
'0.0.15': require(__dirname+'/migration-0.0.15.js'), //messages r9k option '0.0.15': require(__dirname+'/migration-0.0.15.js'), //messages r9k option
'0.0.16': require(__dirname+'/migration-0.0.16.js'), //separate tph/pph triggers '0.0.16': require(__dirname+'/migration-0.0.16.js'), //separate tph/pph triggers
'0.0.17': require(__dirname+'/migration-0.0.17.js'), //add custompages collection '0.0.17': require(__dirname+'/migration-0.0.17.js'), //add custompages collection
'0.0.18': require(__dirname+'/migration-0.0.18.js'), //disable onion file posting to disable anonymizer file posting
'0.0.19': require(__dirname+'/migration-0.0.19.js'), //fix incorrect index causing duplicate key error
} }

@ -3,5 +3,5 @@
module.exports = async(db, redis) => { module.exports = async(db, redis) => {
console.log('add collection for board custompages'); console.log('add collection for board custompages');
await db.createCollection('custompages'); await db.createCollection('custompages');
await db.collection('custompages').createIndex({ 'board': 1, 'url': 1 }, { unique: true }); await db.collection('custompages').createIndex({ 'board': 1, 'page': 1 }, { unique: true });
}; };

@ -0,0 +1,12 @@
'use strict';
module.exports = async(db, redis) => {
console.log('Renaming disable onion file posting to disable anonymizer file posting');
await db.collection('boards').updateMany({}, {
'$rename': {
'settings.disableOnionFilePosting' : 'settings.disableAnonymizerFilePosting',
}
});
console.log('Cleared boards cache');
await redis.deletePattern('board:*');
};

@ -0,0 +1,11 @@
'use strict';
module.exports = async(db, redis) => {
console.log('fixing index for custompages');
try {
await db.collection('custompages').dropIndex('board_1_url_1');
} catch (e) {
// didnt have the bad index
}
await db.collection('custompages').createIndex({ 'board': 1, 'page': 1 }, { unique: true });
};

@ -116,7 +116,7 @@ module.exports = async (req, res, next) => {
'tags': arraySetting(req.body.tags, oldSettings.tags, 10), 'tags': arraySetting(req.body.tags, oldSettings.tags, 10),
'filters': arraySetting(req.body.filters, oldSettings.filters, 50), 'filters': arraySetting(req.body.filters, oldSettings.filters, 50),
'blockedCountries': req.body.countries || [], 'blockedCountries': req.body.countries || [],
'disableOnionFilePosting': booleanSetting(req.body.disable_onion_file_posting), 'disableAnonymizerFilePosting': booleanSetting(req.body.disable_anonymizer_file_posting),
'strictFiltering': booleanSetting(req.body.strict_filtering), 'strictFiltering': booleanSetting(req.body.strict_filtering),
'customCss': globalLimits.customCss.enabled ? (req.body.custom_css !== null ? req.body.custom_css : oldSettings.customCss) : null, 'customCss': globalLimits.customCss.enabled ? (req.body.custom_css !== null ? req.body.custom_css : oldSettings.customCss) : null,
'announcement': { 'announcement': {

@ -455,7 +455,7 @@ ${res.locals.numFiles > 0 ? req.files.file.map(f => f.name+'|'+(f.phash || '')).
}); });
} }
const postId = await Posts.insertOne(res.locals.board, data, thread, res.locals.tor); const postId = await Posts.insertOne(res.locals.board, data, thread, res.locals.anonymizer);
let enableCaptcha = false; //make this returned from some function, refactor and move the next section to another file let enableCaptcha = false; //make this returned from some function, refactor and move the next section to another file
const pphTriggerActive = (pphTriggerAction > 0 && pphTrigger > 0); const pphTriggerActive = (pphTriggerAction > 0 && pphTrigger > 0);

@ -16,7 +16,7 @@ module.exports = async (req, res, next) => {
let captchaId; let captchaId;
let maxAge = 5*60*1000; let maxAge = 5*60*1000;
try { try {
if (!res.locals.tor) { if (!res.locals.anonymizer) {
const ratelimit = await Ratelimits.incrmentQuota(res.locals.ip.single, 'captcha', rateLimitCost.captcha); const ratelimit = await Ratelimits.incrmentQuota(res.locals.ip.single, 'captcha', rateLimitCost.captcha);
if (ratelimit > 100) { if (ratelimit > 100) {
return res.status(429).redirect('/file/ratelimit.png'); return res.status(429).redirect('/file/ratelimit.png');

8
package-lock.json generated

@ -3448,11 +3448,11 @@
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
"version": "4.1.1", "version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": { "requires": {
"ms": "^2.1.1" "ms": "2.1.2"
} }
}, },
"ms": { "ms": {

@ -1,7 +1,7 @@
{ {
"name": "jschan", "name": "jschan",
"version": "0.0.1", "version": "0.0.1",
"migrateVersion": "0.0.17", "migrateVersion": "0.0.19",
"description": "", "description": "",
"main": "server.js", "main": "server.js",
"dependencies": { "dependencies": {

@ -6,7 +6,7 @@ process
const timeUtils = require(__dirname+'/../helpers/timeutils.js') const timeUtils = require(__dirname+'/../helpers/timeutils.js')
, Mongo = require(__dirname+'/../db/db.js') , Mongo = require(__dirname+'/../db/db.js')
, { pruneImmediately, debugLogs, enableWebring } = require(__dirname+'/../configs/main.js') , { pruneIps, pruneImmediately, debugLogs, enableWebring } = require(__dirname+'/../configs/main.js')
, doInterval = require(__dirname+'/../helpers/dointerval.js'); , doInterval = require(__dirname+'/../helpers/dointerval.js');
(async () => { (async () => {
@ -37,6 +37,11 @@ const timeUtils = require(__dirname+'/../helpers/timeutils.js')
doInterval(pruneFiles, timeUtils.DAY, true); doInterval(pruneFiles, timeUtils.DAY, true);
} }
if (pruneIps) {
const ipSchedule = require(__dirname+'/ips.js');
doInterval(ipSchedule, timeUtils.DAY, true);
}
//update the webring //update the webring
if (enableWebring) { if (enableWebring) {
const updateWebring = require(__dirname+'/webring.js'); const updateWebring = require(__dirname+'/webring.js');

@ -0,0 +1,53 @@
'use strict';
/*
prune IPs from old posts (actually, rehash them with a temporary random salt to maintain
post history and prevent *-by-ip action unintentionally deleting many posts)
NOTE: ips may still remain in the following collections:
- bans, because bans need the IP to function
- modlog actioner ips, modlogs are already auto-pruned
- ratelimits, these only last 1 minute
- stats, these last max of 24 hours
*/
const Mongo = require(__dirname+'/../db/db.js')
, { Posts } = require(__dirname+'/../db/')
, { createHash, randomBytes } = require('crypto')
, { pruneIps } = require(__dirname+'/../configs/main.js');
module.exports = async (days) => {
const beforeDate = new Date();
beforeDate.setDate(beforeDate.getDate() - days);
const beforeDateMongoId = Mongo.ObjectId.createFromTime(Math.floor(beforeDate.getTime()/1000));
const tempIpHashSecret = randomBytes(20).toString('base64');
const bulkWrites = [];
await Posts.db.find({
_id: {
$lte: beforeDateMongoId,
},
'ip.pruned': {
$ne: true
}
}).forEach(post => {
const randomIP = createHash('sha256').update(tempIpHashSecret + post.ip.single).digest('base64');
bulkWrites.push({
updateOne: {
filter: {
_id: post._id,
},
update: {
$set: {
'ip.pruned': true,
'ip.raw': randomIP,
'ip.single': randomIP,
'ip.qrange': randomIP,
'ip.hrange': randomIP,
}
}
}
});
});
console.log(`Randomising ip on ${bulkWrites.length} posts`);
if (bulkWrites.length.length > 0) {
await Posts.db.bulkWrite(bulkWrites);
}
}

@ -42,9 +42,9 @@ module.exports = {
|| roomName === 'manage-recent-raw') { || roomName === 'manage-recent-raw') {
requiredAuth = 3; //board mod minimum requiredAuth = 3; //board mod minimum
if (user && authLevel === 4) { if (user && authLevel === 4) {
if (user.ownedBoards.includes(board)) { if (user.ownedBoards.includes(roomBoard)) {
authLevel = 2; //user is BO authLevel = 2; //user is BO
} else if (user.modBoards.includes(board)) { } else if (user.modBoards.includes(roomBoard)) {
authLevel = 3; //user is mod authLevel = 3; //user is mod
} }
} }

@ -299,7 +299,7 @@ block content
p Trigger Reset Lock Mode: If a trigger threshold was reached, reset the lock mode to this at the end of the hour. p Trigger Reset Lock Mode: If a trigger threshold was reached, reset the lock mode to this at the end of the hour.
p Trigger Reset Captcha Mode: If a trigger threshold was reached, reset the captcha mode to this at the end of the hour. p Trigger Reset Captcha Mode: If a trigger threshold was reached, reset the captcha mode to this at the end of the hour.
p Early 404: When a new thread is posted, delete any existing threads with less than #{early404Replies} replies beyond the first 1/#{early404Fraction} of threads. p Early 404: When a new thread is posted, delete any existing threads with less than #{early404Replies} replies beyond the first 1/#{early404Fraction} of threads.
p Disable .onion file posting: Prevent users posting through a .onion hidden service posting images. p Disable anonymizer file posting: Prevent users posting images through anonymizers such as Tor hidden services, lokinet SNApps or i2p eepsites.
p Blocked Countries: Block country codes (based on geo Ip data) from posting. p Blocked Countries: Block country codes (based on geo Ip data) from posting.
p Filters: Newline separated list of words or phrases to match in posts. Checks name, message, filenames, subject, and filenames. p Filters: Newline separated list of words or phrases to match in posts. Checks name, message, filenames, subject, and filenames.
p Strict Filtering: More aggressively match filters, by normalising the input compared against the filters. p Strict Filtering: More aggressively match filters, by normalising the input compared against the filters.

@ -257,9 +257,9 @@ block content
label.postform-style.ph-5 label.postform-style.ph-5
input(type='checkbox', name='early404', value='true' checked=board.settings.early404) input(type='checkbox', name='early404', value='true' checked=board.settings.early404)
.row .row
.label Disable .onion file posting .label Disable anonymizer file posting
label.postform-style.ph-5 label.postform-style.ph-5
input(type='checkbox', name='disable_onion_file_posting', value='true' checked=board.settings.disableOnionFilePosting) input(type='checkbox', name='disable_anonymizer_file_posting', value='true' checked=board.settings.disableAnonymizerFilePosting)
.row .row
.label Blocked Countries .label Blocked Countries
include ../includes/2charisocountries.pug include ../includes/2charisocountries.pug

Loading…
Cancel
Save