fatchan
/
jschan
mirror of https://gitgud.io/fatchan/jschan.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

changes to pruning/dedupe files

Browse Source
merge-requests/208/head
fatchan 5 years ago
parent 627ccbc8cd
commit c5f0049584
3 changed files with 63 additions and 157 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 210
      gulpfile.js
  2. 6
      models/forms/makepost.js
  3. 4
      schedules.js

210
gulpfile.js
Unescape View File

@ -27,164 +27,70 @@ async function wipe() {
, Posts = require(__dirname+'/db/posts.js')
, Bans = require(__dirname+'/db/bans.js')
, Captchas = require(__dirname+'/db/captchas.js')
, Accounts = require(__dirname+'/db/accounts.js');
, Accounts = require(__dirname+'/db/accounts.js')
, Files = require(__dirname+'/db/files.js');
//wipe db shit
await Captchas.deleteAll();
await Accounts.deleteAll();
await Posts.deleteAll();
await Boards.deleteAll();
await Bans.deleteAll();
await Files.deleteAll();
await Promise.all([
Captchas.deleteAll(),
Accounts.deleteAll(),
Posts.deleteAll(),
Boards.deleteAll(),
Bans.deleteAll(),
Files.deleteAll()
]);
//add boards
await Boards.insertOne({
_id: 'file',
owner: '',
moderators: [],
banners: [],
sequence_value: 1,
settings: {
name: 'file',
description: 'testing new file types',
captchaTrigger: 10,
captcha: false,
forceAnon: false,
ids: false,
userPostDelete: true,
userPostSpoiler: true,
userPostUnlink: true,
threadLimit: 200,
replyLimit: 500,
maxFiles: 3,
forceOPSubject: false,
forceOPMessage: true,
forceOPFile: true,
minMessageLength: 0,
defaultName: 'Anonymous',
}
})
await Boards.insertOne({
_id: 'pol',
owner: '',
moderators: [],
banners: [],
sequence_value: 1,
settings: {
name: 'politics',
description: 'talk about politics',
captchaTrigger: 10,
captcha: false,
forceAnon: true,
ids: true,
userPostDelete: true,
userPostSpoiler: true,
userPostUnlink: true,
threadLimit: 200,
replyLimit: 500,
maxFiles: 3,
forceOPSubject: true,
forceOPMessage: true,
forceOPFile: true,
minMessageLength: 0,
defaultName: 'Anonymous',
}
})
await Boards.insertOne({
_id: 'b',
owner: '',
moderators: [],
banners: [],
sequence_value: 1,
settings: {
name: 'random',
description: 'anything and everything',
captchaTrigger: 10,
captcha: false,
forceAnon: false,
ids: false,
userPostDelete: true,
userPostSpoiler: true,
userPostUnlink: true,
threadLimit: 200,
replyLimit: 500,
maxFiles: 3,
forceOPSubject: false,
forceOPMessage: true,
forceOPFile: true,
minMessageLength: 0,
defaultName: 'Anonymous',
}
})
await Boards.insertOne({
_id: 't',
owner: '',
moderators: [],
banners: [],
sequence_value: 1,
settings: {
name: 'test',
description: 'testing board',
captchaTrigger: 10,
captcha: false,
forceAnon: true,
ids: false,
userPostDelete: true,
userPostSpoiler: true,
userPostUnlink: true,
threadLimit: 200,
replyLimit: 500,
maxFiles: 0,
forceOPSubject: false,
forceOPMessage: true,
forceOPFile: false,
minMessageLength: 0,
defaultName: 'Anonymous',
}
})
//add indexes - should profiled and changed at some point if necessary
await Bans.db.dropIndexes();
await Captchas.captcha.dropIndexes();
await Captchas.ratelimit.dropIndexes();
await Posts.db.dropIndexes();
await Files.db.createIndex({ 'count': 1 })
await Bans.db.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 0 }); //custom expiry, i.e. it will expire when current date > than this date
await Captchas.captcha.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 300 }); //captchas valid for 5 minutes
await Captchas.ratelimit.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 60 }); //per minute captcha ratelimit
await Posts.db.createIndex({
'postId': 1,
'board': 1,
});
await Posts.db.createIndex({
'board': 1,
'thread': 1,
'bumped': -1
});
await Posts.db.createIndex({
'board': 1,
'reports.0': 1
}, {
partialFilterExpression: {
'reports.0': {
'$exists': true
}
}
});
await Posts.db.createIndex({
'globalreports.0': 1
}, {
partialFilterExpression: {
'globalreports.0': {
'$exists': true
await Promise.all([
//add test boards
Boards.insertOne({
'_id': 'test',
'owner': '',
'moderators': [],
'banners': [],
'sequence_value': 1,
'settings': {
'name': 'test',
'description': 'testing board',
'captchaMode': 0,
'captchaTrigger': 10,
'captchaTriggerMode': 2,
'forceAnon': true,
'ids': false,
'userPostDelete': true,
'userPostSpoiler': true,
'userPostUnlink': true,
'threadLimit': 200,
'replyLimit': 500,
'maxFiles': 0,
'forceOPSubject': false,
'forceOPMessage': true,
'forceOPFile': false,
'minMessageLength': 0,
'defaultName': 'Anonymous',
'announcement': {
'raw':null,
'markdown':null
},
'filters':[]
}
}
});
//default admin acc
await Accounts.insertOne('admin', 'changeme', 0);
})
//add indexes - should profiled and changed at some point if necessary
, Bans.db.dropIndexes()
, Captchas.captcha.dropIndexes()
, Captchas.ratelimit.dropIndexes()
, Posts.db.dropIndexes()
, Files.db.createIndex({ 'count': 1 })
, Bans.db.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 0 }) //custom expiry, i.e. it will expire when current date > than this date
, Captchas.captcha.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 300 }) //captchas valid for 5 minutes
, Captchas.ratelimit.createIndex({ "expireAt": 1 }, { expireAfterSeconds: 60 }) //per minute captcha ratelimit
, Posts.db.createIndex({ 'postId': 1,'board': 1,})
, Posts.db.createIndex({ 'board': 1, 'thread': 1, 'bumped': -1 })
, Posts.db.createIndex({ 'board': 1, 'reports.0': 1 }, { 'partialFilterExpression': { 'reports.0': { '$exists': true } } })
, Posts.db.createIndex({ 'globalreports.0': 1 }, { 'partialFilterExpression': { 'globalreports.0': { '$exists': true } } })
//default admin acc
, Accounts.insertOne('admin', 'changeme', 0)
]);
Mongo.client.close();
//delete all the static files

6
models/forms/makepost.js
Unescape View File

@ -112,9 +112,6 @@ module.exports = async (req, res, next) => {
let extension = path.extname(file.name) || file.name.substring(file.name.indexOf('.'));
const filename = file.sha256 + extension;
await Files.increment([filename]);
//TODO: reduce increments for failed file uploads, potentially just add it to the deleteTempFiles handler
//get metadata
let processedFile = {
hash: file.sha256,
@ -192,6 +189,9 @@ module.exports = async (req, res, next) => {
}
}
await Files.increment(files.map(file => file.filename));
// because express middleware is autistic i need to do this
deleteTempFiles(req).catch(e => console.error);

4
schedules.js
Unescape View File

@ -45,6 +45,7 @@ async function deleteCaptchas() {
}
}, msTime.minute*5); //rebuild homepage for pph updates
//could make this use a db changefeed
setInterval(async () => {
try {
await deleteCaptchas();
@ -55,6 +56,7 @@ async function deleteCaptchas() {
setInterval(async () => {
try {
//todo: would need to lock the DB or at least disable posting very shortly for this pruning
const files = await Files.db.find({
'count': {
'$lte': 0
@ -64,8 +66,6 @@ async function deleteCaptchas() {
}).toArray().then(res => {
return res.map(x => x._id);
});
//todo: race condition where file is posted after this db call, causing file to be deleted that was just posted
//maybe this is just dumb and i should lock the database temporarily for this?
await Files.db.removeMany({
'count': {
'$lte': 0

Write Preview
Loading…
Cancel
Save