From c5f00495841ffee4c4ac460aeeb1473159dbf0ea Mon Sep 17 00:00:00 2001 From: fatchan Date: Fri, 2 Aug 2019 08:51:56 +0000 Subject: [PATCH] changes to pruning/dedupe files --- gulpfile.js | 210 +++++++++++---------------------------- models/forms/makepost.js | 6 +- schedules.js | 4 +- 3 files changed, 63 insertions(+), 157 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 27d2c5b8..fb3d64d7 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -27,164 +27,70 @@ async function wipe() { , Posts = require(__dirname+'/db/posts.js') , Bans = require(__dirname+'/db/bans.js') , Captchas = require(__dirname+'/db/captchas.js') - , Accounts = require(__dirname+'/db/accounts.js'); + , Accounts = require(__dirname+'/db/accounts.js') , Files = require(__dirname+'/db/files.js'); //wipe db shit - await Captchas.deleteAll(); - await Accounts.deleteAll(); - await Posts.deleteAll(); - await Boards.deleteAll(); - await Bans.deleteAll(); - await Files.deleteAll(); + await Promise.all([ + Captchas.deleteAll(), + Accounts.deleteAll(), + Posts.deleteAll(), + Boards.deleteAll(), + Bans.deleteAll(), + Files.deleteAll() + ]); //add boards - await Boards.insertOne({ - _id: 'file', - owner: '', - moderators: [], - banners: [], - sequence_value: 1, - settings: { - name: 'file', - description: 'testing new file types', - captchaTrigger: 10, - captcha: false, - forceAnon: false, - ids: false, - userPostDelete: true, - userPostSpoiler: true, - userPostUnlink: true, - threadLimit: 200, - replyLimit: 500, - maxFiles: 3, - forceOPSubject: false, - forceOPMessage: true, - forceOPFile: true, - minMessageLength: 0, - defaultName: 'Anonymous', - } - }) - await Boards.insertOne({ - _id: 'pol', - owner: '', - moderators: [], - banners: [], - sequence_value: 1, - settings: { - name: 'politics', - description: 'talk about politics', - captchaTrigger: 10, - captcha: false, - forceAnon: true, - ids: true, - userPostDelete: true, - userPostSpoiler: true, - userPostUnlink: true, - threadLimit: 200, - replyLimit: 500, - maxFiles: 3, - forceOPSubject: true, - forceOPMessage: true, - forceOPFile: true, - minMessageLength: 0, - defaultName: 'Anonymous', - } - }) - await Boards.insertOne({ - _id: 'b', - owner: '', - moderators: [], - banners: [], - sequence_value: 1, - settings: { - name: 'random', - description: 'anything and everything', - captchaTrigger: 10, - captcha: false, - forceAnon: false, - ids: false, - userPostDelete: true, - userPostSpoiler: true, - userPostUnlink: true, - threadLimit: 200, - replyLimit: 500, - maxFiles: 3, - forceOPSubject: false, - forceOPMessage: true, - forceOPFile: true, - minMessageLength: 0, - defaultName: 'Anonymous', - } - }) - await Boards.insertOne({ - _id: 't', - owner: '', - moderators: [], - banners: [], - sequence_value: 1, - settings: { - name: 'test', - description: 'testing board', - captchaTrigger: 10, - captcha: false, - forceAnon: true, - ids: false, - userPostDelete: true, - userPostSpoiler: true, - userPostUnlink: true, - threadLimit: 200, - replyLimit: 500, - maxFiles: 0, - forceOPSubject: false, - forceOPMessage: true, - forceOPFile: false, - minMessageLength: 0, - defaultName: 'Anonymous', - } - }) - - //add indexes - should profiled and changed at some point if necessary - await Bans.db.dropIndexes(); - await Captchas.captcha.dropIndexes(); - await Captchas.ratelimit.dropIndexes(); - await Posts.db.dropIndexes(); - - await Files.db.createIndex({ 'count': 1 }) - await Bans.db.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 0 }); //custom expiry, i.e. it will expire when current date > than this date - await Captchas.captcha.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 300 }); //captchas valid for 5 minutes - await Captchas.ratelimit.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 60 }); //per minute captcha ratelimit - await Posts.db.createIndex({ - 'postId': 1, - 'board': 1, - }); - await Posts.db.createIndex({ - 'board': 1, - 'thread': 1, - 'bumped': -1 - }); - await Posts.db.createIndex({ - 'board': 1, - 'reports.0': 1 - }, { - partialFilterExpression: { - 'reports.0': { - '$exists': true - } - } - }); - await Posts.db.createIndex({ - 'globalreports.0': 1 - }, { - partialFilterExpression: { - 'globalreports.0': { - '$exists': true + await Promise.all([ + //add test boards + Boards.insertOne({ + '_id': 'test', + 'owner': '', + 'moderators': [], + 'banners': [], + 'sequence_value': 1, + 'settings': { + 'name': 'test', + 'description': 'testing board', + 'captchaMode': 0, + 'captchaTrigger': 10, + 'captchaTriggerMode': 2, + 'forceAnon': true, + 'ids': false, + 'userPostDelete': true, + 'userPostSpoiler': true, + 'userPostUnlink': true, + 'threadLimit': 200, + 'replyLimit': 500, + 'maxFiles': 0, + 'forceOPSubject': false, + 'forceOPMessage': true, + 'forceOPFile': false, + 'minMessageLength': 0, + 'defaultName': 'Anonymous', + 'announcement': { + 'raw':null, + 'markdown':null + }, + 'filters':[] } - } - }); - - //default admin acc - await Accounts.insertOne('admin', 'changeme', 0); + }) + //add indexes - should profiled and changed at some point if necessary + , Bans.db.dropIndexes() + , Captchas.captcha.dropIndexes() + , Captchas.ratelimit.dropIndexes() + , Posts.db.dropIndexes() + , Files.db.createIndex({ 'count': 1 }) + , Bans.db.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 0 }) //custom expiry, i.e. it will expire when current date > than this date + , Captchas.captcha.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 300 }) //captchas valid for 5 minutes + , Captchas.ratelimit.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 60 }) //per minute captcha ratelimit + , Posts.db.createIndex({ 'postId': 1,'board': 1,}) + , Posts.db.createIndex({ 'board': 1, 'thread': 1, 'bumped': -1 }) + , Posts.db.createIndex({ 'board': 1, 'reports.0': 1 }, { 'partialFilterExpression': { 'reports.0': { '$exists': true } } }) + , Posts.db.createIndex({ 'globalreports.0': 1 }, { 'partialFilterExpression': { 'globalreports.0': { '$exists': true } } }) + //default admin acc + , Accounts.insertOne('admin', 'changeme', 0) + ]); Mongo.client.close(); //delete all the static files diff --git a/models/forms/makepost.js b/models/forms/makepost.js index 3c72c348..b8d6c15e 100644 --- a/models/forms/makepost.js +++ b/models/forms/makepost.js @@ -112,9 +112,6 @@ module.exports = async (req, res, next) => { let extension = path.extname(file.name) || file.name.substring(file.name.indexOf('.')); const filename = file.sha256 + extension; - await Files.increment([filename]); -//TODO: reduce increments for failed file uploads, potentially just add it to the deleteTempFiles handler - //get metadata let processedFile = { hash: file.sha256, @@ -192,6 +189,9 @@ module.exports = async (req, res, next) => { } } + + await Files.increment(files.map(file => file.filename)); + // because express middleware is autistic i need to do this deleteTempFiles(req).catch(e => console.error); diff --git a/schedules.js b/schedules.js index bb98e86c..42636f75 100644 --- a/schedules.js +++ b/schedules.js @@ -45,6 +45,7 @@ async function deleteCaptchas() { } }, msTime.minute*5); //rebuild homepage for pph updates + //could make this use a db changefeed setInterval(async () => { try { await deleteCaptchas(); @@ -55,6 +56,7 @@ async function deleteCaptchas() { setInterval(async () => { try { +//todo: would need to lock the DB or at least disable posting very shortly for this pruning const files = await Files.db.find({ 'count': { '$lte': 0 @@ -64,8 +66,6 @@ async function deleteCaptchas() { }).toArray().then(res => { return res.map(x => x._id); }); - //todo: race condition where file is posted after this db call, causing file to be deleted that was just posted - //maybe this is just dumb and i should lock the database temporarily for this? await Files.db.removeMany({ 'count': { '$lte': 0