fatchan
/
jschan
mirror of https://gitgud.io/fatchan/jschan.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

no more lmx. since we have redis, use it for redlock and connect-redis for sessions instead of mongo

Browse Source
merge-requests/208/head
fatchan 5 years ago
parent 99b738753f
commit d899a31bbe
14 changed files with 88 additions and 218 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 24
      README.md
  2. 15
      ecosystem.config.js
  3. 2
      helpers/files/imageidentify.js
  4. 2
      helpers/files/imagethumbnail.js
  5. 2
      helpers/files/imageupload.js
  6. 6
      helpers/render.js
  7. 20
      models/forms/makepost.js
  8. 8
      mutex.js
  9. 161
      package-lock.json
  10. 9
      package.json
  11. 9
      redlock.js
  12. 23
      schedules.js
  13. 20
      server.js
  14. 5
      worker.js

24
README.md
Unescape View File

@ -21,16 +21,16 @@ Demo site running at https://fatpeople.lol
- [x] Customise homepage, faq, rules or add custom pages
## Todo
- [ ] Post moving/thread merging
- [ ] Flags. Geographic and custom uploaded
- [ ] IP range bans
- [ ] IP notes/records/ban history of some sort
- [ ] JSON api
- [ ] Configuration editor
- [ ] Overboard/multiboard/meta boards
- [ ] Boards search page
- [ ] User created board custom pages
- [ ] File URL uploads
- Post moving/thread merging
- Geographic and custom uploaded flags
- IP range bans
- IP notes/records/ban history of some sort
- JSON api
- Configuration editor
- Overboard/multiboard/meta boards
- Boards search page
- User created board custom pages
- File URL uploads
## Setup
Please note:
@ -38,10 +38,10 @@ Please note:
- For debian.
##### Requirements
- Linux (most likely could work elsewhere, but why?)
- Linux
- Node.js (to run the app)
- MongoDB (database, duh)
- Redis (queues, and eventually for caching and mutex/locking)
- Redis (sessions, queue, locks and caching)
- Nginx (handle https, serve static content and html)
- Certbot/letsencrypt (for https cert)
- Imagemagick (thumbnailing images)

15
ecosystem.config.js
Unescape View File

@ -1,21 +1,6 @@
module.exports = {
// Options reference: https://pm2.io/doc/en/runtime/reference/ecosystem-file/
apps : [{
name: 'lock-broker',
script: 'node_modules/live-mutex/dist/lm-start-server.js',
args: '--use-uds',
instances: 1,
autorestart: true,
watch: false,
max_memory_restart: '1G',
log_date_format: 'YYYY-MM-DD HH:mm:ss.SSS',
env: {
NODE_ENV: 'development'
},
env_production: {
NODE_ENV: 'production'
}
}, {
name: 'build-worker',
script: 'worker.js',
instances: 1, //could increase if building is getting backed up

2
helpers/files/imageidentify.js
Unescape View File

@ -1,4 +1,4 @@
const gm = require('@tohru/gm')
const gm = require('gm')
, configs = require(__dirname+'/../../configs/main.json')
, uploadDirectory = require(__dirname+'/uploadDirectory.js');

2
helpers/files/imagethumbnail.js
Unescape View File

@ -1,4 +1,4 @@
const gm = require('@tohru/gm')
const gm = require('gm')
, configs = require(__dirname+'/../../configs/main.json')
, uploadDirectory = require(__dirname+'/uploadDirectory.js');

2
helpers/files/imageupload.js
Unescape View File

@ -1,7 +1,7 @@
'use strict';
const uploadDirectory = require(__dirname+'/uploadDirectory.js')
, gm = require('@tohru/gm');
, gm = require('gm');
module.exports = (file, filename, folder) => {

6
helpers/render.js
Unescape View File

@ -5,13 +5,13 @@ const { cacheTemplates, meta }= require(__dirname+'/../configs/main.json')
, pug = require('pug')
, path = require('path')
, uploadDirectory = require(__dirname+'/files/uploadDirectory.js')
, Mutex = require(__dirname+'/../mutex.js')
, redlock = require(__dirname+'/../redlock.js')
, templateDirectory = path.join(__dirname+'/../views/pages/');
module.exports = async (htmlName, templateName, options) => {
const html = pug.renderFile(`${templateDirectory}${templateName}`, { ...options, cache: cacheTemplates, meta });
const { id, key } = await Mutex.acquire(htmlName);
const lock = await redlock.lock(`locks:${htmlName}`, 3000); //what is a reasonable ttl?
await outputFile(`${uploadDirectory}html/${htmlName}`, html);
await Mutex.release(key, id);
await lock.unlock();
return html;
};

20
models/forms/makepost.js
Unescape View File

@ -446,16 +446,16 @@ module.exports = async (req, res, next) => {
const prunedThreads = await Posts.pruneThreads(res.locals.board);
if (prunedThreads.length > 0) {
await deletePosts(prunedThreads, req.params.board);
if (!enableCaptcha) {
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': Math.ceil(threadLimit/10)
}
});
}
}
if (!enableCaptcha) {
buildQueue.push({
'task': 'buildBoardMultiple',
'options': {
'board': res.locals.board,
'startpage': 1,
'endpage': Math.ceil(threadLimit/10)
}
});
}
}

8
mutex.js
Unescape View File

@ -1,8 +0,0 @@
'use strict';
const { Client } = require('live-mutex');
const mutexClient = new Client({ udsPath: process.env.HOME + '/.lmx/uds.sock' });
mutexClient.emitter.on('warning', console.error);
module.exports = mutexClient;

161
package-lock.json generated
Unescape View File

@ -39,19 +39,6 @@
}
}
},
"@oresoftware/json-stream-parser": {
"version": "0.0.113",
"resolved": "https://registry.npmjs.org/@oresoftware/json-stream-parser/-/json-stream-parser-0.0.113.tgz",
"integrity": "sha512-In7ufVanxBaCA7kxSRDtvkg5z/BInb5SneBhGPq+1UpRRqPUQ3KHECKKU3fdApHgPjnfbjGUQFCDaWvGVawraA=="
},
"@oresoftware/linked-queue": {
"version": "0.1.105",
"resolved": "https://registry.npmjs.org/@oresoftware/linked-queue/-/linked-queue-0.1.105.tgz",
"integrity": "sha512-XpYqg7BM39s9sF+1RSP7fpNNRsVNW847k+X8iiO26K939XeT4BHxG7ZBtwUVOVYI+T3JDjm1wcvGT2/pYH/7Jg==",
"requires": {
"chalk": "^2.4.2"
}
},
"@pm2/agent": {
"version": "0.5.26",
"resolved": "https://registry.npmjs.org/@pm2/agent/-/agent-0.5.26.tgz",
@ -193,32 +180,6 @@
}
}
},
"@tohru/gm": {
"version": "git+https://github.com/fatchan/gm.git#07df8fbf131b6b18c32fa010a84e67964e132955",
"from": "git+https://github.com/fatchan/gm.git",
"requires": {
"array-parallel": "^0.1.3",
"array-series": "^0.1.5",
"cross-spawn": "^4.0.0",
"debug": "^3.1.0",
"tmp": "^0.0.33"
},
"dependencies": {
"debug": {
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"@types/babel-types": {
"version": "7.0.7",
"resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.7.tgz",
@ -870,6 +831,11 @@
"resolved": "https://registry.npmjs.org/blessed/-/blessed-0.1.81.tgz",
"integrity": "sha1-+WLWh+wsNpVwrnGvhDJW5tDKESk="
},
"bluebird": {
"version": "3.5.5",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz",
"integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w=="
},
"bodec": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/bodec/-/bodec-0.1.0.tgz",
@ -935,11 +901,6 @@
}
}
},
"bson": {
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/bson/-/bson-1.0.9.tgz",
"integrity": "sha512-IQX9/h7WdMBIW/q/++tGd+emQr0XMdeZ6icnT/74Xk9fnabWn+gZgpE+9V+gujL3hhJOoNrnDVY7tWdzc7NUTg=="
},
"buffer-equal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
@ -950,11 +911,6 @@
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"buffer-shims": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz",
"integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E="
},
"bull": {
"version": "3.10.0",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.10.0.tgz",
@ -1102,9 +1058,9 @@
}
},
"chownr": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
"integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz",
"integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A=="
},
"class-utils": {
"version": "0.3.6",
@ -1345,52 +1301,10 @@
"source-map": "^0.6.1"
}
},
"connect-mongo": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/connect-mongo/-/connect-mongo-2.0.3.tgz",
"integrity": "sha512-Vs+QZ/6X6gbCrP1Ls7Oh/wlyY6pgpbPSrUKF5yRT+zd+4GZPNbjNquxquZ+Clv2+03HBXE7T4lVM0PUcaBhihg==",
"requires": {
"mongodb": "^2.0.36"
},
"dependencies": {
"mongodb": {
"version": "2.2.36",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.36.tgz",
"integrity": "sha512-P2SBLQ8Z0PVx71ngoXwo12+FiSfbNfGOClAao03/bant5DgLNkOPAck5IaJcEk4gKlQhDEURzfR3xuBG1/B+IA==",
"requires": {
"es6-promise": "3.2.1",
"mongodb-core": "2.1.20",
"readable-stream": "2.2.7"
}
},
"process-nextick-args": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
"integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
},
"readable-stream": {
"version": "2.2.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.7.tgz",
"integrity": "sha1-BwV6y+JGeyIELTb5jFrVBwVOlbE=",
"requires": {
"buffer-shims": "~1.0.0",
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "~1.0.0",
"process-nextick-args": "~1.0.6",
"string_decoder": "~1.0.0",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
"integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
"connect-redis": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-4.0.0.tgz",
"integrity": "sha512-yCSSSCcE/AwHH66o3bMa991Hs2aL/FqtlvVjkBYTHpbefeVlwRImQEIMnK550D9ZEnikhkHttlICnjAPbRK56w=="
},
"console-control-strings": {
"version": "1.1.0",
@ -1971,11 +1885,6 @@
"es6-symbol": "^3.1.1"
}
},
"es6-promise": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.2.1.tgz",
"integrity": "sha1-7FYjOGgDKQkgcXDDlEjiREndH8Q="
},
"es6-promisify": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
@ -4234,18 +4143,6 @@
"resolve": "^1.1.7"
}
},
"live-mutex": {
"version": "0.1.1066",
"resolved": "https://registry.npmjs.org/live-mutex/-/live-mutex-0.1.1066.tgz",
"integrity": "sha512-k61Ubi9lY+MatIUqCb14NFxun0BB7GUFjBwhxAR6/YzPf0h8zoKkZWwkPKUAZT4cWxLhX18ZNsZf/gNya+Dn2g==",
"requires": {
"@oresoftware/json-stream-parser": "0.0.113",
"@oresoftware/linked-queue": "0.1.105",
"chalk": "^2.4.2",
"tcp-ping": "^0.1.1",
"uuid": "^3.3.2"
}
},
"load-json-file": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
@ -4516,9 +4413,9 @@
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
},
"minipass": {
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
"integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.5.0.tgz",
"integrity": "sha512-9FwMVYhn6ERvMR8XFdOavRz4QK/VJV8elU1x50vYexf9lslDcWe/f4HBRxCPd185ekRSjU6CfYyJCECa/CQy7Q==",
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@ -4601,15 +4498,6 @@
}
}
},
"mongodb-core": {
"version": "2.1.20",
"resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-2.1.20.tgz",
"integrity": "sha512-IN57CX5/Q1bhDq6ShAR6gIv4koFsZP7L8WOK1S0lR0pVDQaScffSMV5jxubLsmZ7J+UdqmykKw4r9hG3XQEGgQ==",
"requires": {
"bson": "~1.0.4",
"require_optional": "~1.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -4764,9 +4652,9 @@
"integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g=="
},
"npm-packlist": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz",
"integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==",
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.4.tgz",
"integrity": "sha512-zTLo8UcVYtDU3gdeaFu2Xu0n0EvelfHDGuqtNIn5RO7yQj4H1TqNdBc/yZjxnWA0PVB8D3Woyp0i5B43JwQ6Vw==",
"requires": {
"ignore-walk": "^3.0.1",
"npm-bundled": "^1.0.1"
@ -5824,6 +5712,14 @@
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redlock": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/redlock/-/redlock-4.0.0.tgz",
"integrity": "sha512-971JQ2rBzZKIOrlqjjcEO4lbxmuq71QtfTNYk7w/m9h59mBYpN+r7xHZEWZw8ubEwvrxxzmqOl4fC2o05+UjDw==",
"requires": {
"bluebird": "^3.3.3"
}
},
"regenerator-runtime": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
@ -6550,11 +6446,6 @@
}
}
},
"tcp-ping": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/tcp-ping/-/tcp-ping-0.1.1.tgz",
"integrity": "sha1-At1/QrW/fXy3jVt6rO+hVf2PfAw="
},
"terser": {
"version": "3.17.0",
"resolved": "https://registry.npmjs.org/terser/-/terser-3.17.0.tgz",

9
package.json
Unescape View File

@ -4,11 +4,10 @@
"description": "",
"main": "server.js",
"dependencies": {
"@tohru/gm": "git+https://github.com/fatchan/gm.git",
"bcrypt": "^3.0.6",
"body-parser": "^1.19.0",
"bull": "^3.10.0",
"connect-mongo": "^2.0.3",
"connect-redis": "^4.0.0",
"convert-svg-to-png": "^0.5.0",
"cookie-parser": "^1.4.4",
"csurf": "^1.10.0",
@ -27,19 +26,17 @@
"gulp-pug": "^4.0.1",
"gulp-uglify-es": "^1.0.4",
"ioredis": "^4.14.0",
"live-mutex": "^0.1.1066",
"lodash": "^4.17.15",
"lodash.mergewith": "^4.6.2",
"mongodb": "^3.3.0",
"path": "^0.12.7",
"pm2": "^3.5.1",
"pug": "^2.0.4",
"redlock": "^4.0.0",
"sanitize-html": "^1.20.1",
"saslprep": "^1.0.3"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"setup": "npm i -g pm2 gulp",
"setup": "npm i -g pm2 gulp && gulp",
"start": "pm2 start ecosystem.config.js --env production"
},
"author": "",

9
redlock.js
Unescape View File

@ -0,0 +1,9 @@
'use strict';
const Redlock = require('redlock')
, { redisClient } = require(__dirname+'/redis.js')
, redlock = new Redlock([redisClient]);
redlock.on('clientError', console.error);
module.exports = redlock;

23
schedules.js
Unescape View File

@ -7,31 +7,34 @@ process
const msTime = require(__dirname+'/helpers/mstime.js')
, deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js')
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js')
, buildQueue = require(__dirname+'/queue.js');
(async () => {
console.log('CONNECTING TO MONGODB');
await Mongo.connect();
await Mutex.connect();
const Files = require(__dirname+'/db/files.js');
console.log('Starting schedules');
console.log('STARTING SCHEDULES');
setInterval(async () => {
buildQueue.push({
'task': 'buildHomepage',
'options': {}
})
}, msTime.minute*5); //rebuild homepage for pph updates
//add 5 minute repeatable job to queue (queue will prevent duplicate)
buildQueue.push({
'task': 'buildHomepage',
'options': {}
}, {
'repeat': {
'cron': '*/5 * * * *'
}
});
//delete files for expired captchas
setInterval(async () => {
try {
await deleteCaptchas();
} catch (e) {
console.error(e);
}
}, msTime.minute*5); //delete files for expired captchas
}, msTime.minute*5);
setInterval(async () => {
try {

20
server.js
Unescape View File

@ -6,7 +6,7 @@ process
const express = require('express')
, session = require('express-session')
, MongoStore = require('connect-mongo')(session)
, redisStore = require('connect-redis')(session)
, path = require('path')
, app = express()
, bodyParser = require('body-parser')
@ -14,22 +14,17 @@ const express = require('express')
, configs = require(__dirname+'/configs/main.json')
, ipHash = require(__dirname+'/helpers/iphash.js')
, referrerCheck = require(__dirname+'/helpers/referrercheck.js')
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js');
, Mongo = require(__dirname+'/db/db.js');
(async () => {
console.log('STARTING IN MODE:', process.env.NODE_ENV);
//connect to mongodb
// connect to mongodb
console.log('CONNECTING TO MONGODB');
await Mongo.connect();
//use live mutex for locking, will switch to redis later
console.log('CONNECTING TO LMX');
await Mutex.connect();
//connect to redis
// connect to redis
console.log('CONNECTING TO REDIS');
const { redisClient } = require(__dirname+'/redis.js');
@ -38,15 +33,14 @@ const express = require('express')
// parse forms (is json required?)
app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
//parse cookies
// parse cookies
app.use(cookieParser());
// session store
app.use(session({
secret: configs.sessionSecret,
store: new MongoStore({
db: Mongo.client.db('sessions'),
stringify: false //keep sessions as object in db
store: new redisStore({
client: redisClient,
}),
resave: false,
saveUninitialized: false,

5
worker.js
Unescape View File

@ -6,13 +6,12 @@ process
const Queue = require('bull')
, configs = require(__dirname+'/configs/main.json')
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js');
, Mongo = require(__dirname+'/db/db.js');
(async () => {
console.log('CONNECTING TO MONGODB');
await Mongo.connect();
await Mutex.connect();
const buildTasks = require(__dirname+'/helpers/build.js')
, generateQueue = new Queue('generate', { 'redis': configs.redis });

Write Preview
Loading…
Cancel
Save