fatchan
/
jschan
mirror of https://gitgud.io/fatchan/jschan.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

temp file delete helper, add unlink_file to actionchecker, backlinks in build -- might remove backlinking but works fine for now

Browse Source
merge-requests/208/head
fatchan 5 years ago
parent 6b549c1539
commit f23e4cd20e
8 changed files with 215 additions and 56 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 86
      build.js
  2. 28
      controllers/forms.js
  3. 3
      helpers/actionchecker.js
  4. 11
      helpers/files/deletetempfiles.js
  5. 10
      helpers/quotes.js
  6. 55
      models/forms/make-post.js
  7. 77
      package-lock.json
  8. 1
      package.json

86
build.js
Unescape View File

@ -28,6 +28,34 @@ module.exports = {
if (!thread) {
return; //this thread may have been an OP that was deleted
}
/*
temporary, jsut seeing how well this works
*/
const postMap = new Map()
postMap.set(thread.postId, thread)
for (let i = 0; i < thread.replies.length; i++) {
const reply = thread.replies[i];
postMap.set(reply.postId, reply);
}
for (let i = 0; i < thread.replies.length; i++) {
const reply = thread.replies[i];
if (!reply.quotes) continue;
for (let j = 0; j < reply.quotes.length; j++) {
const quote = reply.quotes[j];
if (postMap.has(quote)) {
const post = postMap.get(quote)
if (!post.backlinks) {
post.backlinks = [];
}
post.backlinks.push(reply.postId);
}
}
}
/*
temporary, jsut seeing how well this works
*/
return render(`${board._id}/thread/${threadId}.html`, 'thread.pug', {
board,
thread
@ -40,6 +68,35 @@ module.exports = {
if (!maxPage) {
maxPage = Math.ceil((await Posts.getPages(board._id)) / 10);
}
/*
temporary, jsut seeing how well this works
*/
for (let k = 0; k < threads.length; k++) {
const thread = threads[k];
const postMap = new Map()
postMap.set(thread.postId, thread)
for (let i = 0; i < thread.replies.length; i++) {
const reply = thread.replies[i];
postMap.set(reply.postId, reply);
}
for (let i = 0; i < thread.replies.length; i++) {
const reply = thread.replies[i];
if (!reply.quotes) continue;
for (let j = 0; j < reply.quotes.length; j++) {
const quote = reply.quotes[j];
if (postMap.has(quote)) {
const post = postMap.get(quote)
if (!post.backlinks) {
post.backlinks = [];
}
post.backlinks.push(reply.postId);
}
}
}
}
/*
temporary, jsut seeing how well this works
*/
return render(`${board._id}/${page === 1 ? 'index' : page}.html`, 'board.pug', {
board,
threads,
@ -60,6 +117,35 @@ module.exports = {
}
const difference = endpage-startpage + 1; //+1 because for single pagemust be > 0
const threads = await Posts.getRecent(board._id, startpage, difference*10);
/*
temporary, jsut seeing how well this works
*/
for (let k = 0; k < threads.length; k++) {
const thread = threads[k];
const postMap = new Map()
postMap.set(thread.postId, thread)
for (let i = 0; i < thread.replies.length; i++) {
const reply = thread.replies[i];
postMap.set(reply.postId, reply);
}
for (let i = 0; i < thread.replies.length; i++) {
const reply = thread.replies[i];
if (!reply.quotes) continue;
for (let j = 0; j < reply.quotes.length; j++) {
const quote = reply.quotes[j];
if (postMap.has(quote)) {
const post = postMap.get(quote)
if (!post.backlinks) {
post.backlinks = [];
}
post.backlinks.push(reply.postId);
}
}
}
}
/*
temporary, jsut seeing how well this works
*/
const buildArray = [];
for (let i = startpage; i <= endpage; i++) {
//console.log('multi building board page', `${board._id}/${i === 1 ? 'index' : i}.html`);

28
controllers/forms.js
Unescape View File

@ -13,15 +13,14 @@ const express = require('express')
, loginAccount = require(__dirname+'/../models/forms/login.js')
, changePassword = require(__dirname+'/../models/forms/changepassword.js')
, registerAccount = require(__dirname+'/../models/forms/register.js')
, deleteTempFiles = require(__dirname+'/../helpers/files/deletetempfiles.js')
, checkPermsMiddleware = require(__dirname+'/../helpers/haspermsmiddleware.js')
, checkPerms = require(__dirname+'/../helpers/hasperms.js')
, paramConverter = require(__dirname+'/../helpers/paramconverter.js')
, banCheck = require(__dirname+'/../helpers/bancheck.js')
, deletePostFiles = require(__dirname+'/../helpers/files/deletepostfiles.js')
, verifyCaptcha = require(__dirname+'/../helpers/captchaverify.js')
, actionHandler = require(__dirname+'/../models/forms/actionhandler.js')
, csrf = require(__dirname+'/../helpers/csrfmiddleware.js')
, deleteFailedFiles = require(__dirname+'/../helpers/files/deletefailed.js')
, actionChecker = require(__dirname+'/../helpers/actionchecker.js');
@ -201,6 +200,9 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verif
}
if (errors.length > 0) {
if (numFiles > 0) {
await deleteTempFiles(req.files.file)
}
return res.status(400).render('message', {
'title': 'Bad request',
'errors': errors,
@ -211,14 +213,8 @@ router.post('/board/:board/post', Boards.exists, banCheck, paramConverter, verif
try {
await makePost(req, res, next, numFiles);
} catch (err) {
//handler errors here better
if (numFiles > 0) {
const fileNames = []
for (let i = 0; i < req.files.file.length; i++) {
remove(req.files.file[i].tempFilePath).catch(e => console.error);
fileNames.push(req.files.file[i].filename);
}
deletePostFiles(fileNames).catch(err => console.error);
await deleteTempFiles(req.files.file)
}
return next(err);
}
@ -276,11 +272,14 @@ router.post('/board/:board/addbanners', csrf, Boards.exists, checkPermsMiddlewar
if (numFiles === 0) {
errors.push('Must provide a file');
}
if (res.locals.board.banners.length > 100) {
errors.push('Limit of 100 banners reached');
if (res.locals.board.banners.length+numFiles > 100) {
errors.push('Number of uploads would exceed 100 banner limit');
}
if (errors.length > 0) {
if (numFiles > 0) {
await deleteTempFiles(req.files.file)
}
return res.status(400).render('message', {
'title': 'Bad request',
'errors': errors,
@ -291,14 +290,9 @@ router.post('/board/:board/addbanners', csrf, Boards.exists, checkPermsMiddlewar
try {
await uploadBanners(req, res, next, numFiles);
} catch (err) {
const fileNames = [];
if (numFiles > 0) {
for (let i = 0; i < req.files.file.length; i++) {
remove(req.files.file[i].tempFilePath).catch(e => console.error);
fileNames.push(req.files.file[i].filename);
}
await deleteTempFiles(req.files.file)
}
deleteFailedFiles(fileNames, 'banner').catch(e => console.error);
return next(err);
}

3
helpers/actionchecker.js
Unescape View File

@ -1,7 +1,8 @@
'use strict';
const actions = [
{name:'delete_file', global:true, auth:false, passwords:true},
{name:'unlink_file', global:true, auth:false, passwords:true},
{name:'delete_file', global:true, auth:true, passwords:false},
{name:'spoiler', global:true, auth:false, passwords:true},
{name:'delete', global:true, auth:false, passwords:true},
{name:'lock', global:false, auth:true, passwords:false},

11
helpers/files/deletetempfiles.js
Unescape View File

@ -0,0 +1,11 @@
'use strict';
const remove = require('fs-extra').remove;
module.exports = async (files) => {
return Promise.all(files.map(async file => {
remove(file.tempFilePath);
}));
}

10
helpers/quotes.js
Unescape View File

@ -11,14 +11,14 @@ module.exports = async (board, text) => {
const quotes = text.match(quoteRegex);
const crossQuotes = text.match(crossQuoteRegex);
if (!quotes && !crossQuotes) {
return text;
return { quotedMessage: text, threadQuotes: [] };
}
//make query for db including crossquotes
const queryOrs = []
const crossQuoteMap = {};
if (quotes) {
const quoteIds = quotes.map(q => +q.substring(2));
const quoteIds = [...new Set(quotes.map(q => +q.substring(2)))]; //only uniques
queryOrs.push({
'board': board,
'postId': {
@ -58,7 +58,7 @@ module.exports = async (board, text) => {
const posts = await Posts.getPostsForQuotes(queryOrs);
//if none of the quotes were real, dont do a replace
if (posts.length === 0) {
return text;
return { quotedMessage: text, threadQuotes: [] };
}
//turn the result into a map of postId => threadId/postId
for (let i = 0; i < posts.length; i++) {
@ -71,10 +71,12 @@ module.exports = async (board, text) => {
}
//then replace the quotes with only ones that exist
const threadQuotes = [];
if (quotes && Object.keys(postThreadIdMap).length > 0) {
text = text.replace(quoteRegex, (match) => {
const quotenum = +match.substring(2);
if (postThreadIdMap[board] && postThreadIdMap[board][quotenum]) {
threadQuotes.push(quotenum)
return `<a class='quote' href='/${board}/thread/${postThreadIdMap[board][quotenum]}.html#${quotenum}'>&gt;&gt;${quotenum}</a>`;
}
return match;
@ -94,6 +96,6 @@ module.exports = async (board, text) => {
});
}
return text;
return { quotedMessage: text, threadQuotes };
}

55
models/forms/make-post.js
Unescape View File

@ -1,11 +1,10 @@
'use strict';
const uuidv4 = require('uuid/v4')
, path = require('path')
const path = require('path')
, util = require('util')
, crypto = require('crypto')
, randomBytes = util.promisify(crypto.randomBytes)
, remove = require('fs-extra').remove
, { remove, pathExists } = require('fs-extra')
, uploadDirectory = require(__dirname+'/../../helpers/uploadDirectory.js')
, Posts = require(__dirname+'/../../db/posts.js')
, getTripCode = require(__dirname+'/../../helpers/tripcode.js')
@ -88,40 +87,43 @@ module.exports = async (req, res, next, numFiles) => {
// then upload, thumb, get metadata, etc.
for (let i = 0; i < numFiles; i++) {
const file = req.files.file[i];
const uuid = uuidv4();
const filename = uuid + path.extname(file.name);
const filename = file.sha256 + path.extname(file.name);
file.filename = filename; //for error to delete failed files
//get metadata
let processedFile = {
hash: file.sha256,
filename: filename,
originalFilename: file.name,
mimetype: file.mimetype,
size: file.size,
};
//check if already exists
const existsFull = await pathExists(`${uploadDirectory}img/${filename}`);
const existsThumb = await pathExists(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`);
//handle video/image ffmpeg or graphicsmagick
const mainType = file.mimetype.split('/')[0];
switch (mainType) {
case 'image':
await imageUpload(file, filename, 'img');
const imageData = await imageIdentify(filename, 'img');
const imageData = await imageIdentify(req.files.file[i].tempFilePath, null, true);
processedFile.geometry = imageData.size // object with width and height pixels
processedFile.sizeString = formatSize(processedFile.size) // 123 Ki string
processedFile.geometryString = imageData.Geometry // 123 x 123 string
if (fileCheckMimeType(file.mimetype, {image: true}) //always thumbnail gif/webp
processedFile.hasThumb = !(fileCheckMimeType(file.mimetype, {image: true})
&& processedFile.geometry.height <= 128
&& processedFile.geometry.width <= 128) {
processedFile.hasThumb = false;
} else {
processedFile.hasThumb = true;
&& processedFile.geometry.width <= 128);
if (!existsFull) {
await imageUpload(file, filename, 'img');
}
if (!existsThumb && processedFile.hasThumb) {
await imageThumbnail(filename);
}
break;
case 'video':
//video metadata
await videoUpload(file, filename, 'img');
const videoData = await videoIdentify(filename);
const videoData = await videoIdentify(req.files.file[i].tempFilePath, null, true);
videoData.streams = videoData.streams.filter(stream => stream.width != null); //filter to only video streams or something with a resolution
processedFile.duration = videoData.format.duration;
processedFile.durationString = new Date(videoData.format.duration*1000).toLocaleString('en-US', {hour12:false}).split(' ')[1].replace(/^00:/, '');
@ -129,10 +131,15 @@ module.exports = async (req, res, next, numFiles) => {
processedFile.sizeString = formatSize(processedFile.size) // 123 Ki string
processedFile.geometryString = `${processedFile.geometry.width}x${processedFile.geometry.height}` // 123 x 123 string
processedFile.hasThumb = true;
await videoThumbnail(filename);
if (!existsFull) {
await videoUpload(file, filename, 'img');
}
if (!existsThumb) {
await videoThumbnail(filename);
}
break;
default:
return next(new Error(`invalid file mime type: ${mainType}`));
throw new Error(`invalid file mime type: ${mainType}`); //throw so goes to error handler before next'ing
}
//delete the temp file
@ -197,9 +204,12 @@ module.exports = async (req, res, next, numFiles) => {
//simple markdown and sanitize
let message = req.body.message;
let quotes = [];
if (message && message.length > 0) {
message = simpleMarkdown(req.params.board, req.body.thread, message);
message = await linkQuotes(req.params.board, message);
const { quotedMessage, threadQuotes } = await linkQuotes(req.params.board, message);
message = quotedMessage;
quotes = threadQuotes;
message = sanitize(message, sanitizeOptions);
}
@ -223,6 +233,7 @@ module.exports = async (req, res, next, numFiles) => {
files,
'reports': [],
'globalreports': [],
quotes
}
if (!req.body.thread) {
@ -248,7 +259,7 @@ module.exports = async (req, res, next, numFiles) => {
if (data.thread) {
//refersh pages
const threadPage = await Posts.getThreadPage(req.params.board, thread);
if (data.email === 'sage') {
if (data.email === 'sage' || thread.sage) {
//refresh the page that the thread is on
parallelPromises.push(buildBoard(res.locals.board, threadPage));
} else {
@ -259,6 +270,14 @@ module.exports = async (req, res, next, numFiles) => {
//new thread, rebuild all pages and prunes old threads
const prunedThreads = await Posts.pruneOldThreads(req.params.board, res.locals.board.settings.threadLimit);
for (let i = 0; i < prunedThreads.length; i++) {
//TODO: consider:
//should i keep these? as an "archive" since they are removed from the DB
//posting wouldnt work and it would just be served as a static file
//files dont matter
//or i could add and set to "archive:true" with the same affect as locking
//but additionally does not appear in board index/catalog but allows to be rebuilt for template updates?
//or is a first party archive kinda against the point of an imageboard?
//i feel like threads epiring and not existing anymore is part of the design
parallelPromises.push(remove(`${uploadDirectory}html/${req.params.board}/thread/${prunedThreads[i]}.html`));
}
parallelPromises.push(buildBoardMultiple(res.locals.board, 1, 10));

77
package-lock.json generated
Unescape View File

@ -396,7 +396,8 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"dev": true
"dev": true,
"optional": true
},
"assign-symbols": {
"version": "1.0.0",
@ -582,6 +583,14 @@
}
}
},
"basic-auth": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz",
"integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==",
"requires": {
"safe-buffer": "5.1.2"
}
},
"bcrypt": {
"version": "3.0.6",
"resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-3.0.6.tgz",
@ -961,6 +970,7 @@
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
"dev": true,
"optional": true,
"requires": {
"delayed-stream": "~1.0.0"
}
@ -1355,7 +1365,8 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true
"dev": true,
"optional": true
},
"delegates": {
"version": "1.0.0",
@ -1857,7 +1868,8 @@
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
"dev": true
"dev": true,
"optional": true
},
"fancy-log": {
"version": "1.3.3",
@ -2136,7 +2148,8 @@
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"aproba": {
"version": "1.2.0",
@ -2157,12 +2170,14 @@
"balanced-match": {
"version": "1.0.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -2177,17 +2192,20 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"concat-map": {
"version": "0.0.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"core-util-is": {
"version": "1.0.2",
@ -2304,7 +2322,8 @@
"inherits": {
"version": "2.0.3",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"ini": {
"version": "1.3.5",
@ -2316,6 +2335,7 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -2330,6 +2350,7 @@
"version": "3.0.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -2337,12 +2358,14 @@
"minimist": {
"version": "0.0.8",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"minipass": {
"version": "2.3.5",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@ -2361,6 +2384,7 @@
"version": "0.5.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -2441,7 +2465,8 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"object-assign": {
"version": "4.1.1",
@ -2453,6 +2478,7 @@
"version": "1.4.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -2538,7 +2564,8 @@
"safe-buffer": {
"version": "5.1.2",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"safer-buffer": {
"version": "2.1.2",
@ -2574,6 +2601,7 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -2593,6 +2621,7 @@
"version": "3.0.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -2636,12 +2665,14 @@
"wrappy": {
"version": "1.0.2",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"yallist": {
"version": "3.0.3",
"bundled": true,
"dev": true
"dev": true,
"optional": true
}
}
},
@ -3552,7 +3583,8 @@
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
"dev": true
"dev": true,
"optional": true
},
"json-schema": {
"version": "0.2.3",
@ -4070,6 +4102,18 @@
"require_optional": "~1.0.0"
}
},
"morgan": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/morgan/-/morgan-1.9.1.tgz",
"integrity": "sha512-HQStPIV4y3afTiCYVxirakhlCfGkI161c76kKFca7Fk1JusM//Qeo1ej2XaMniiNeaZklMVrh3vTtIzpzwbpmA==",
"requires": {
"basic-auth": "~2.0.0",
"debug": "2.6.9",
"depd": "~1.1.2",
"on-finished": "~2.3.0",
"on-headers": "~1.0.1"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -5815,7 +5859,8 @@
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
"dev": true
"dev": true,
"optional": true
},
"type-is": {
"version": "1.6.16",

1
package.json
Unescape View File

@ -18,6 +18,7 @@
"fs-extra": "^7.0.1",
"helmet": "^3.16.0",
"mongodb": "^3.2.3",
"morgan": "^1.9.1",
"path": "^0.12.7",
"pug": "^2.0.3",
"sanitize-html": "^1.20.0",

Write Preview
Loading…
Cancel
Save