diff --git a/configs/main.json.example b/configs/main.json.example index ab1d4790..5346939a 100644 --- a/configs/main.json.example +++ b/configs/main.json.example @@ -1,11 +1,15 @@ { "dbURL": "mongodb://username:password@host:port", + "redis": { + "host": "127.0.0.1", + "port": "6379", + "password": "long random string" + }, "port": 7000, "sessionSecret": "long random string", "tripcodeSecret": "long random string", "ipHashSecret": "long random string", "postPasswordSecret": "long random string", - "redisPassword": "long random string", "cacheTemplates": true, "refererCheck": false, "refererRegex": "^https?:\\/\\/(?:www\\.)?domain\\.com\\/", diff --git a/package-lock.json b/package-lock.json index ed8a451b..7fda8285 100644 --- a/package-lock.json +++ b/package-lock.json @@ -262,14 +262,6 @@ "resolved": "https://registry.npmjs.org/@types/pug/-/pug-2.0.4.tgz", "integrity": "sha1-h3L80EGOPNLMFxVV1zAHQVBR9LI=" }, - "@types/redis": { - "version": "2.8.13", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.13.tgz", - "integrity": "sha512-p86cm5P6DMotUqCS6odQRz0JJwc5QXZw9eyH0ALVIqmq12yqtex5ighWyGFHKxak9vaA/GF/Ilu0KZ0MuXXUbg==", - "requires": { - "@types/node": "*" - } - }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -963,6 +955,30 @@ "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", "integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" }, + "bull": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/bull/-/bull-3.10.0.tgz", + "integrity": "sha512-LbQsc7c+eYd7IaJD7tS373yKLYttjTfoPZ+9xYYlPM5+gutAjofSTsESOGGyaxyX2lE1dkg+eWhUK5kAPl5Zow==", + "requires": { + "cron-parser": "^2.7.3", + "debuglog": "^1.0.0", + "get-port": "^5.0.0", + "ioredis": "^4.5.1", + "lodash": "^4.17.11", + "p-timeout": "^3.1.0", + "promise.prototype.finally": "^3.1.0", + "semver": "^6.1.1", + "util.promisify": "^1.0.0", + "uuid": "^3.2.1" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, "busboy": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", @@ -1498,6 +1514,15 @@ "moment-timezone": "^0.5.x" } }, + "cron-parser": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.13.0.tgz", + "integrity": "sha512-UWeIpnRb0eyoWPVk+pD3TDpNx3KCFQeezO224oJIkktBrcW6RoAPOx5zIKprZGfk6vcYSmA8yQXItejSaDBhbQ==", + "requires": { + "is-nan": "^1.2.1", + "moment-timezone": "^0.5.25" + } + }, "cross-spawn": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", @@ -1578,6 +1603,11 @@ "ms": "2.0.0" } }, + "debuglog": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", + "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=" + }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", @@ -1796,11 +1826,6 @@ "domelementtype": "1" } }, - "double-ended-queue": { - "version": "2.1.0-0", - "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", - "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" - }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -1903,6 +1928,29 @@ "is-arrayish": "^0.2.1" } }, + "es-abstract": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", + "integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==", + "requires": { + "es-to-primitive": "^1.2.0", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "is-callable": "^1.1.4", + "is-regex": "^1.0.4", + "object-keys": "^1.0.12" + } + }, + "es-to-primitive": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", + "integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, "es5-ext": { "version": "0.10.50", "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.50.tgz", @@ -3050,6 +3098,14 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==" }, + "get-port": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.0.0.tgz", + "integrity": "sha512-imzMU0FjsZqNa6BqOjbbW6w5BivHIuQKopjpPqcnx0AVHJQKCxK1O+Ab3OrVXhrekqfVMjwA9ZYu062R+KcIsQ==", + "requires": { + "type-fest": "^0.3.0" + } + }, "get-stdin": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz", @@ -3778,6 +3834,11 @@ "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, + "is-callable": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", + "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==" + }, "is-data-descriptor": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", @@ -3786,6 +3847,11 @@ "kind-of": "^3.0.2" } }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" + }, "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", @@ -3845,6 +3911,14 @@ "is-extglob": "^2.1.1" } }, + "is-nan": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.2.1.tgz", + "integrity": "sha1-n69ltvttskt/XAYoR16nH5iEAeI=", + "requires": { + "define-properties": "^1.1.1" + } + }, "is-negated-glob": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-negated-glob/-/is-negated-glob-1.0.0.tgz", @@ -3908,6 +3982,14 @@ "is-unc-path": "^1.0.0" } }, + "is-symbol": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", + "integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==", + "requires": { + "has-symbols": "^1.0.0" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -4793,6 +4875,15 @@ "isobject": "^3.0.0" } }, + "object.getownpropertydescriptors": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz", + "integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=", + "requires": { + "define-properties": "^1.1.2", + "es-abstract": "^1.5.1" + } + }, "object.map": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object.map/-/object.map-1.0.1.tgz", @@ -4919,11 +5010,24 @@ "os-tmpdir": "^1.0.0" } }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, "p-map": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" }, + "p-timeout": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.1.0.tgz", + "integrity": "sha512-C27DYI+tCroT8J8cTEyySGydl2B7FlxrGNF5/wmMbl1V+jeehUCzEE/BVgzRebdm2K3ZitKOKx8YbdFumDyYmw==", + "requires": { + "p-finally": "^1.0.0" + } + }, "pac-proxy-agent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-3.0.0.tgz", @@ -5333,6 +5437,16 @@ "asap": "~2.0.3" } }, + "promise.prototype.finally": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.1.tgz", + "integrity": "sha512-gnt8tThx0heJoI3Ms8a/JdkYBVhYP/wv+T7yQimR+kdOEJL21xTFbiJhMRqnSPcr54UVvMbsscDk2w+ivyaLPw==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.13.0", + "function-bind": "^1.1.1" + } + }, "promptly": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/promptly/-/promptly-2.2.0.tgz", @@ -5700,16 +5814,6 @@ "resolve": "^1.1.6" } }, - "redis": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", - "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", - "requires": { - "double-ended-queue": "^2.1.0-0", - "redis-commands": "^1.2.0", - "redis-parser": "^2.6.0" - } - }, "redis-commands": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", @@ -5720,11 +5824,6 @@ "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" }, - "redis-parser": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", - "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" - }, "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", @@ -5929,16 +6028,6 @@ "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz", "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w=" }, - "rsmq": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/rsmq/-/rsmq-0.11.0.tgz", - "integrity": "sha512-Uo+jQ85T3jbajAwoU7MGyIVRC1ytUDfMV+yjskQzIMEJDLWpcJmw+DlacM3EXwjwD9g4Q2yWdKuGXi8C2V8WIg==", - "requires": { - "@types/redis": "^2.8.0", - "lodash": "^4.17.11", - "redis": "^2.8.0" - } - }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -6652,6 +6741,11 @@ "prelude-ls": "~1.1.2" } }, + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==" + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -6846,6 +6940,15 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "util.promisify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz", + "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==", + "requires": { + "define-properties": "^1.1.2", + "object.getownpropertydescriptors": "^2.0.3" + } + }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/package.json b/package.json index 2a317102..6a6504c6 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "@tohru/gm": "git+https://github.com/fatchan/gm.git", "bcrypt": "^3.0.6", "body-parser": "^1.19.0", + "bull": "^3.10.0", "connect-mongo": "^2.0.3", "convert-svg-to-png": "^0.5.0", "cookie-parser": "^1.4.4", @@ -33,7 +34,6 @@ "path": "^0.12.7", "pm2": "^3.5.1", "pug": "^2.0.4", - "rsmq": "^0.11.0", "sanitize-html": "^1.20.1", "saslprep": "^1.0.3" }, diff --git a/queue.js b/queue.js index ed8196a7..93911eea 100644 --- a/queue.js +++ b/queue.js @@ -1,19 +1,9 @@ -const RedisSMQ = require('rsmq') - , { redisClient } = require(__dirname+'/redis.js') - , rsmq = new RedisSMQ({ ns: 'rsmq', client: redisClient }) - , queuename = 'generate' +'use strict'; -rsmq.createQueue({ qname: queuename }, (err) => { - if (err && err.name !== 'queueExists') { - return console.error(err); - } -}); +const Queue = require('bull') + , configs = require(__dirname+'/configs/main.json') + , generateQueue = new Queue('generate', { 'redis': configs.redis }); module.exports.push = (data) => { - rsmq.sendMessage({ qname: queuename, message: JSON.stringify(data) }, (err) => { - if (err) { - return console.error(err); - } - //message enqueued successfully - }); + generateQueue.add(data); } diff --git a/redis.js b/redis.js index 4623a79c..da822fc9 100644 --- a/redis.js +++ b/redis.js @@ -6,7 +6,7 @@ const Redis = require('ioredis') module.exports = { - redisClient, //the redis client instance (So we can share with queues elsewhere) + redisClient, //cache not used yet, but will need to JSON stringify things that are objects e.g. boards, threads get: async (key) => { diff --git a/server.js b/server.js index 7507938c..2f5f3bb0 100644 --- a/server.js +++ b/server.js @@ -29,11 +29,9 @@ const express = require('express') console.log('CONNECTING TO LMX'); await Mutex.connect(); - //redis for queue, and in future for caching, so moving the client instance here. + //connect to redis console.log('CONNECTING TO REDIS'); - //need an instance of redis client to disconnect cleanly on server close const { redisClient } = require(__dirname+'/redis.js'); - //connecting is not async, so just requiring it here so logging is correct order // disable useless express header app.disable('x-powered-by'); diff --git a/worker.js b/worker.js index 33e73631..1350dbdd 100644 --- a/worker.js +++ b/worker.js @@ -4,54 +4,24 @@ process .on('uncaughtException', console.error) .on('unhandledRejection', console.error); -const RedisSMQ = require('rsmq') +const Queue = require('bull') , configs = require(__dirname+'/configs/main.json') - , { redisClient } = require(__dirname+'/redis.js') - , rsmq = new RedisSMQ({ ns: 'rsmq', client: redisClient }) - , queuename = 'generate' , Mongo = require(__dirname+'/db/db.js') , Mutex = require(__dirname+'/mutex.js'); -let buildTasks = {} - , interval = 100; - (async () => { await Mongo.connect(); await Mutex.connect(); - buildTasks = require(__dirname+'/helpers/build.js'); - rsmq.createQueue({ qname: queuename }, (err) => { - if (err && err.name !== 'queueExists') { - return console.error(err); - } - setTimeout(processQueueLoop, interval); + const buildTasks = require(__dirname+'/helpers/build.js') + , generateQueue = new Queue('generate', { 'redis': configs.redis }); + + generateQueue.process(async (job, done) => { + await buildTasks[job.data.task](job.data.options); + done(); }); })(); -function processQueueLoop() { - rsmq.receiveMessage({ qname: queuename }, async (err, resp) => { - if (err) { - return console.error(err); - } - if (resp.id) { - interval = 100; //keeps queue checking fast when there are tasks - const message = JSON.parse(resp.message); - await buildTasks[message.task](message.options); - rsmq.deleteMessage({ qname: queuename, id: resp.id }, (err) => { - if (err) { - return console.error(err); - } - //message deleted successfully - }); - } else { - //max 2 sec poll time - if (interval < 2000) { //slow down queue when empty - interval += 100; - } - } - setTimeout(processQueueLoop, interval); - }); -}