fatchan
/
jschan
mirror of https://gitgud.io/fatchan/jschan.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch to bull instead of rsmq its superior in every way

Browse Source
merge-requests/208/head
fatchan 5 years ago
parent 3aabb5dbb5
commit 4e22ee61ce
7 changed files with 161 additions and 96 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      configs/main.json.example
  2. 179
      package-lock.json
  3. 2
      package.json
  4. 20
      queue.js
  5. 2
      redis.js
  6. 4
      server.js
  7. 44
      worker.js

6
configs/main.json.example
Unescape View File

@ -1,11 +1,15 @@
{
"dbURL": "mongodb://username:password@host:port",
"redis": {
"host": "127.0.0.1",
"port": "6379",
"password": "long random string"
},
"port": 7000,
"sessionSecret": "long random string",
"tripcodeSecret": "long random string",
"ipHashSecret": "long random string",
"postPasswordSecret": "long random string",
"redisPassword": "long random string",
"cacheTemplates": true,
"refererCheck": false,
"refererRegex": "^https?:\\/\\/(?:www\\.)?domain\\.com\\/",

179
package-lock.json generated
Unescape View File

@ -262,14 +262,6 @@
"resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.4.tgz",
"integrity": "sha1-h3L80EGOPNLMFxVV1zAHQVBR9LI="
},
"@types/redis": {
"version": "2.8.13",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.13.tgz",
"integrity": "sha512-p86cm5P6DMotUqCS6odQRz0JJwc5QXZw9eyH0ALVIqmq12yqtex5ighWyGFHKxak9vaA/GF/Ilu0KZ0MuXXUbg==",
"requires": {
"@types/node": "*"
}
},
"abbrev": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@ -963,6 +955,30 @@
"resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz",
"integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E="
},
"bull": {
"version": "3.10.0",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.10.0.tgz",
"integrity": "sha512-LbQsc7c+eYd7IaJD7tS373yKLYttjTfoPZ+9xYYlPM5+gutAjofSTsESOGGyaxyX2lE1dkg+eWhUK5kAPl5Zow==",
"requires": {
"cron-parser": "^2.7.3",
"debuglog": "^1.0.0",
"get-port": "^5.0.0",
"ioredis": "^4.5.1",
"lodash": "^4.17.11",
"p-timeout": "^3.1.0",
"promise.prototype.finally": "^3.1.0",
"semver": "^6.1.1",
"util.promisify": "^1.0.0",
"uuid": "^3.2.1"
},
"dependencies": {
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
}
}
},
"busboy": {
"version": "0.2.14",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
@ -1498,6 +1514,15 @@
"moment-timezone": "^0.5.x"
}
},
"cron-parser": {
"version": "2.13.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.13.0.tgz",
"integrity": "sha512-UWeIpnRb0eyoWPVk+pD3TDpNx3KCFQeezO224oJIkktBrcW6RoAPOx5zIKprZGfk6vcYSmA8yQXItejSaDBhbQ==",
"requires": {
"is-nan": "^1.2.1",
"moment-timezone": "^0.5.25"
}
},
"cross-spawn": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz",
@ -1578,6 +1603,11 @@
"ms": "2.0.0"
}
},
"debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI="
},
"decamelize": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
@ -1796,11 +1826,6 @@
"domelementtype": "1"
}
},
"double-ended-queue": {
"version": "2.1.0-0",
"resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz",
"integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw="
},
"duplexify": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@ -1903,6 +1928,29 @@
"is-arrayish": "^0.2.1"
}
},
"es-abstract": {
"version": "1.13.0",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz",
"integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==",
"requires": {
"es-to-primitive": "^1.2.0",
"function-bind": "^1.1.1",
"has": "^1.0.3",
"is-callable": "^1.1.4",
"is-regex": "^1.0.4",
"object-keys": "^1.0.12"
}
},
"es-to-primitive": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz",
"integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==",
"requires": {
"is-callable": "^1.1.4",
"is-date-object": "^1.0.1",
"is-symbol": "^1.0.2"
}
},
"es5-ext": {
"version": "0.10.50",
"resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.50.tgz",
@ -3050,6 +3098,14 @@
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
"integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w=="
},
"get-port": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.0.0.tgz",
"integrity": "sha512-imzMU0FjsZqNa6BqOjbbW6w5BivHIuQKopjpPqcnx0AVHJQKCxK1O+Ab3OrVXhrekqfVMjwA9ZYu062R+KcIsQ==",
"requires": {
"type-fest": "^0.3.0"
}
},
"get-stdin": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz",
@ -3778,6 +3834,11 @@
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
},
"is-callable": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz",
"integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA=="
},
"is-data-descriptor": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
@ -3786,6 +3847,11 @@
"kind-of": "^3.0.2"
}
},
"is-date-object": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
"integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
},
"is-descriptor": {
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
@ -3845,6 +3911,14 @@
"is-extglob": "^2.1.1"
}
},
"is-nan": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.2.1.tgz",
"integrity": "sha1-n69ltvttskt/XAYoR16nH5iEAeI=",
"requires": {
"define-properties": "^1.1.1"
}
},
"is-negated-glob": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-negated-glob/-/is-negated-glob-1.0.0.tgz",
@ -3908,6 +3982,14 @@
"is-unc-path": "^1.0.0"
}
},
"is-symbol": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz",
"integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==",
"requires": {
"has-symbols": "^1.0.0"
}
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@ -4793,6 +4875,15 @@
"isobject": "^3.0.0"
}
},
"object.getownpropertydescriptors": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz",
"integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=",
"requires": {
"define-properties": "^1.1.2",
"es-abstract": "^1.5.1"
}
},
"object.map": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/object.map/-/object.map-1.0.1.tgz",
@ -4919,11 +5010,24 @@
"os-tmpdir": "^1.0.0"
}
},
"p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
"integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
},
"p-map": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
"integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="
},
"p-timeout": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.1.0.tgz",
"integrity": "sha512-C27DYI+tCroT8J8cTEyySGydl2B7FlxrGNF5/wmMbl1V+jeehUCzEE/BVgzRebdm2K3ZitKOKx8YbdFumDyYmw==",
"requires": {
"p-finally": "^1.0.0"
}
},
"pac-proxy-agent": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-3.0.0.tgz",
@ -5333,6 +5437,16 @@
"asap": "~2.0.3"
}
},
"promise.prototype.finally": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.1.tgz",
"integrity": "sha512-gnt8tThx0heJoI3Ms8a/JdkYBVhYP/wv+T7yQimR+kdOEJL21xTFbiJhMRqnSPcr54UVvMbsscDk2w+ivyaLPw==",
"requires": {
"define-properties": "^1.1.3",
"es-abstract": "^1.13.0",
"function-bind": "^1.1.1"
}
},
"promptly": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/promptly/-/promptly-2.2.0.tgz",
@ -5700,16 +5814,6 @@
"resolve": "^1.1.6"
}
},
"redis": {
"version": "2.8.0",
"resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz",
"integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==",
"requires": {
"double-ended-queue": "^2.1.0-0",
"redis-commands": "^1.2.0",
"redis-parser": "^2.6.0"
}
},
"redis-commands": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz",
@ -5720,11 +5824,6 @@
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz",
"integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs="
},
"regenerator-runtime": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
@ -5929,16 +6028,6 @@
"resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz",
"integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w="
},
"rsmq": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/rsmq/-/rsmq-0.11.0.tgz",
"integrity": "sha512-Uo+jQ85T3jbajAwoU7MGyIVRC1ytUDfMV+yjskQzIMEJDLWpcJmw+DlacM3EXwjwD9g4Q2yWdKuGXi8C2V8WIg==",
"requires": {
"@types/redis": "^2.8.0",
"lodash": "^4.17.11",
"redis": "^2.8.0"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -6652,6 +6741,11 @@
"prelude-ls": "~1.1.2"
}
},
"type-fest": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz",
"integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ=="
},
"type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@ -6846,6 +6940,15 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"util.promisify": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz",
"integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==",
"requires": {
"define-properties": "^1.1.2",
"object.getownpropertydescriptors": "^2.0.3"
}
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",

2
package.json
Unescape View File

@ -7,6 +7,7 @@
"@tohru/gm": "git+https://github.com/fatchan/gm.git",
"bcrypt": "^3.0.6",
"body-parser": "^1.19.0",
"bull": "^3.10.0",
"connect-mongo": "^2.0.3",
"convert-svg-to-png": "^0.5.0",
"cookie-parser": "^1.4.4",
@ -33,7 +34,6 @@
"path": "^0.12.7",
"pm2": "^3.5.1",
"pug": "^2.0.4",
"rsmq": "^0.11.0",
"sanitize-html": "^1.20.1",
"saslprep": "^1.0.3"
},

20
queue.js
Unescape View File

@ -1,19 +1,9 @@
const RedisSMQ = require('rsmq')
, { redisClient } = require(__dirname+'/redis.js')
, rsmq = new RedisSMQ({ ns: 'rsmq', client: redisClient })
, queuename = 'generate'
'use strict';
rsmq.createQueue({ qname: queuename }, (err) => {
if (err && err.name !== 'queueExists') {
return console.error(err);
}
});
const Queue = require('bull')
, configs = require(__dirname+'/configs/main.json')
, generateQueue = new Queue('generate', { 'redis': configs.redis });
module.exports.push = (data) => {
rsmq.sendMessage({ qname: queuename, message: JSON.stringify(data) }, (err) => {
if (err) {
return console.error(err);
}
//message enqueued successfully
});
generateQueue.add(data);
}

2
redis.js
Unescape View File

@ -6,7 +6,7 @@ const Redis = require('ioredis')
module.exports = {
redisClient, //the redis client instance (So we can share with queues elsewhere)
redisClient,
//cache not used yet, but will need to JSON stringify things that are objects e.g. boards, threads
get: async (key) => {

4
server.js
Unescape View File

@ -29,11 +29,9 @@ const express = require('express')
console.log('CONNECTING TO LMX');
await Mutex.connect();
//redis for queue, and in future for caching, so moving the client instance here.
//connect to redis
console.log('CONNECTING TO REDIS');
//need an instance of redis client to disconnect cleanly on server close
const { redisClient } = require(__dirname+'/redis.js');
//connecting is not async, so just requiring it here so logging is correct order
// disable useless express header
app.disable('x-powered-by');

44
worker.js
Unescape View File

@ -4,54 +4,24 @@ process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const RedisSMQ = require('rsmq')
const Queue = require('bull')
, configs = require(__dirname+'/configs/main.json')
, { redisClient } = require(__dirname+'/redis.js')
, rsmq = new RedisSMQ({ ns: 'rsmq', client: redisClient })
, queuename = 'generate'
, Mongo = require(__dirname+'/db/db.js')
, Mutex = require(__dirname+'/mutex.js');
let buildTasks = {}
, interval = 100;
(async () => {
await Mongo.connect();
await Mutex.connect();
buildTasks = require(__dirname+'/helpers/build.js');
rsmq.createQueue({ qname: queuename }, (err) => {
if (err && err.name !== 'queueExists') {
return console.error(err);
}
setTimeout(processQueueLoop, interval);
const buildTasks = require(__dirname+'/helpers/build.js')
, generateQueue = new Queue('generate', { 'redis': configs.redis });
generateQueue.process(async (job, done) => {
await buildTasks[job.data.task](job.data.options);
done();
});
})();
function processQueueLoop() {
rsmq.receiveMessage({ qname: queuename }, async (err, resp) => {
if (err) {
return console.error(err);
}
if (resp.id) {
interval = 100; //keeps queue checking fast when there are tasks
const message = JSON.parse(resp.message);
await buildTasks[message.task](message.options);
rsmq.deleteMessage({ qname: queuename, id: resp.id }, (err) => {
if (err) {
return console.error(err);
}
//message deleted successfully
});
} else {
//max 2 sec poll time
if (interval < 2000) { //slow down queue when empty
interval += 100;
}
}
setTimeout(processQueueLoop, interval);
});
}

Write Preview
Loading…
Cancel
Save