diff --git a/db/boards.js b/db/boards.js index 3d8ad2f4..b89c7ff2 100644 --- a/db/boards.js +++ b/db/boards.js @@ -108,16 +108,25 @@ module.exports = { ); }, - frontPageSortLimit: () => { + boardSort: (skip=0, limit=20) => { return db.find({ 'settings.unlisted': { '$ne': true } + }, { + 'projection': { + '_id': 1, + 'sequence_value': 1, + 'pph': 1, + 'ips': 1, + 'settings.description': 1, + 'settings.name': 1, + } }).sort({ 'ips': -1, 'pph': -1, 'sequence_value': -1, - }).limit(20).toArray(); + }).skip(skip).limit(limit).toArray(); }, totalPosts: () => { diff --git a/gulp/res/css/style.css b/gulp/res/css/style.css index 1d204ee6..a0cdc4bc 100644 --- a/gulp/res/css/style.css +++ b/gulp/res/css/style.css @@ -707,24 +707,30 @@ hr + .thread { margin-top: -5px; } -@keyframes spin { - 0% { transform: rotate(0deg); } - 100% { transform: rotate(360deg); } +.webringtable th { + position: sticky; top: 0; } -.loader { - border: 8px solid #00000010; - border-radius: 50%; - border-bottom: 8px solid #00000090; - width: 32px; - height: 32px; - animation: spin 2s linear infinite; +.boardtable.webringtable { + border: none; +} + +.scrolltable { + max-height: 160px; + overflow-y: auto; + overflow-x: hidden; + border: 1px solid var(--box-border-color); +} + +table.boardtable td:nth-child(3),table.boardtable td:nth-child(4),table.boardtable td:nth-child(5), +table.boardtable th:nth-child(3),table.boardtable th:nth-child(4),table.boardtable th:nth-child(5) { + min-width: 80px; } @media only screen and (max-width: 600px) { - table#boardtable td:nth-child(3), table#boardtable th:nth-child(3), - table#boardtable td:nth-child(4), table#boardtable th:nth-child(4) { + table.boardtable td:nth-child(3), table.boardtable th:nth-child(3), + table.boardtable td:nth-child(4), table.boardtable th:nth-child(4) { display: none; } diff --git a/gulpfile.js b/gulpfile.js index 105054a0..1e302f0b 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -57,8 +57,9 @@ async function wipe() { '_id': 'test', 'owner': '', 'banners': [], - 'sequence_value': 1, 'pph': 0, + 'ips': 0, + 'sequence_value': 1, 'settings': { 'name': 'test', 'description': 'testing board', @@ -92,6 +93,7 @@ async function wipe() { //delete all the static files return Promise.all([ del([ 'static/html/*' ]), + del([ 'static/json/*' ]), del([ 'static/banner/*' ]), del([ 'static/captcha/*' ]), del([ 'static/img/*' ]), @@ -114,7 +116,10 @@ function images() { } function deletehtml() { - return del([ 'static/html/*' ]); //these will be now build-on-load + return Promise.all([ + del([ 'static/html/*' ]), + del([ 'static/json/*' ]) + ]); } function custompages() { diff --git a/helpers/build.js b/helpers/build.js index 1e2d036b..86deefbc 100644 --- a/helpers/build.js +++ b/helpers/build.js @@ -1,7 +1,9 @@ 'use strict'; const Mongo = require(__dirname+'/../db/db.js') + , cache = require(__dirname+'/../redis.js') , msTime = require(__dirname+'/mstime.js') + , { enableWebring } = require(__dirname+'/../configs/main.json') , { Posts, Files, Boards, News, Modlogs } = require(__dirname+'/../db/') , render = require(__dirname+'/render.js') , timeDiffString = (label, end) => `${label} -> ${end[0] > 0 ? end[0]+'s ' : ''}${(end[1]/1000000).toFixed(2)}ms`; @@ -219,15 +221,17 @@ module.exports = { if (bulkWrites.length > 0) { await Boards.db.bulkWrite(bulkWrites); } - const [ totalPosts, boards, fileStats ] = await Promise.all([ + const [ totalPosts, boards, webringBoards, fileStats ] = await Promise.all([ Boards.totalPosts(), //overall total posts ever made - Boards.frontPageSortLimit(), //boards sorted by users, pph, total posts + Boards.boardSort(0, 20), //top 20 boards sorted by users, pph, total posts + enableWebring ? cache.get('webring:boards') : null, Files.activeContent() //size of all files ]); const html = render('index.html', 'home.pug', { totalPosts: totalPosts, activeUsers, boards, + webringBoards, fileStats, }); const end = process.hrtime(start); diff --git a/helpers/captcha/deletecaptchas.js b/helpers/captcha/deletecaptchas.js index c80f72a5..a39fde3a 100644 --- a/helpers/captcha/deletecaptchas.js +++ b/helpers/captcha/deletecaptchas.js @@ -15,7 +15,6 @@ module.exports = async () => { const expiry = new Date(stats.ctime).getTime() + msTime.minute*5; if (now > expiry) { await remove(filePath); - console.log(`Deleted expired captcha ${filePath}`) } } catch (e) { console.error(e); diff --git a/helpers/files/prune.js b/helpers/files/prune.js new file mode 100644 index 00000000..2e850182 --- /dev/null +++ b/helpers/files/prune.js @@ -0,0 +1,32 @@ +'use strict'; + +const Files = require(__dirname+'/../../db/files.js') + , { remove } = require('fs-extra') + , uploadDirectory = require(__dirname+'/uploadDirectory.js'); + +module.exports = async() => { + //todo: make this not a race condition, but it only happens daily so ¯\_(ツ)_/¯ + const files = await Files.db.aggregate({ + 'count': { + '$lte': 1 + } + }, { + 'projection': { + 'count': 0, + 'size': 0 + } + }).toArray().then(res => { + return res.map(x => x._id); + }); + await Files.db.removeMany({ + 'count': { + '$lte': 0 + } + }); + await Promise.all(files.map(async filename => { + return Promise.all([ + remove(`${uploadDirectory}img/${filename}`), + remove(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`) + ]) + })); +} diff --git a/models/forms/create.js b/models/forms/create.js index 067e4a91..18e2b01c 100644 --- a/models/forms/create.js +++ b/models/forms/create.js @@ -26,6 +26,7 @@ module.exports = async (req, res, next) => { 'banners': [], 'sequence_value': 1, 'pph': 0, + 'ips': 0, 'settings': { name, description, diff --git a/package-lock.json b/package-lock.json index 7ffa8a5c..9f981fef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4606,6 +4606,11 @@ "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, "node-pre-gyp": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz", diff --git a/package.json b/package.json index 1b2ab84c..28f4c0c3 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "gulp-uglify-es": "^1.0.4", "ioredis": "^4.14.0", "mongodb": "^3.3.2", + "node-fetch": "^2.6.0", "path": "^0.12.7", "pm2": "^3.5.1", "pug": "^2.0.4", diff --git a/schedules.js b/schedules.js index 7f3f8ed9..3394261b 100644 --- a/schedules.js +++ b/schedules.js @@ -5,16 +5,14 @@ process .on('unhandledRejection', console.error); const msTime = require(__dirname+'/helpers/mstime.js') - , deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js') , Mongo = require(__dirname+'/db/db.js') + , { enableWebring } = require(__dirname+'/configs/main.json') , buildQueue = require(__dirname+'/queue.js'); (async () => { console.log('CONNECTING TO MONGODB'); await Mongo.connect(); - const Files = require(__dirname+'/db/files.js'); - console.log('STARTING SCHEDULES'); //add 5 minute repeatable job to queue (queue will prevent duplicate) @@ -28,6 +26,8 @@ const msTime = require(__dirname+'/helpers/mstime.js') }); //delete files for expired captchas + const deleteCaptchas = require(__dirname+'/helpers/captcha/deletecaptchas.js'); + deleteCaptchas().catch(e => console.error); setInterval(async () => { try { await deleteCaptchas(); @@ -36,33 +36,25 @@ const msTime = require(__dirname+'/helpers/mstime.js') } }, msTime.minute*5); + //update webring + if (enableWebring) { + const updateWebring = require(__dirname+'/webring.js'); + updateWebring().catch(e => console.error); + setInterval(async () => { + try { + await updateWebring(); + } catch (e) { + console.error(e); + } + }, msTime.hour); + } + + //file pruning + const pruneFiles = require(__dirname+'/helpers/files/prune.js'); + pruneFiles().catch(e => console.error); setInterval(async () => { try { -//todo: make this not a race condition, but it only happens daily so ¯\_(ツ)_/¯ - const files = await Files.db.aggregate({ - 'count': { - '$lte': 1 - } - }, { - 'projection': { - 'count': 0, - 'size': 0 - } - }).toArray().then(res => { - return res.map(x => x._id); - }); - await Files.db.removeMany({ - 'count': { - '$lte': 0 - } - }); - await Promise.all(files.map(async filename => { - return Promise.all([ - remove(`${uploadDirectory}img/${filename}`), - remove(`${uploadDirectory}img/thumb-${filename.split('.')[0]}.jpg`) - ]) - })); - console.log('Deleted unused files:', files); + await pruneFiles(); } catch (e) { console.error(e); } diff --git a/views/pages/home.pug b/views/pages/home.pug index 761c5f22..d8bc8d6a 100644 --- a/views/pages/home.pug +++ b/views/pages/home.pug @@ -21,32 +21,33 @@ block content | Choose a topic below to join the discussion, or a(href='/create.html') create your own board | . - .table-container.flex-center.mv-10.text-center - table#boardtable - tr - th Board - th Description - th(title='Posts in the last hour') Posts/h - th(title='Unique IPs that have posted in the last 72h') Active Users - th Total Posts - each board in boards + if boards && boards.length > 0 + .table-container.flex-center.mv-10.text-center + table.boardtable tr - td: a(href=`/${board._id}/`) /#{board._id}/ - #{board.settings.name} - td #{board.settings.description} - td #{board.pph} - td #{board.ips} - td #{board.sequence_value-1} - .table-container.flex-center.mv-10.text-center - table - tr - th Total Posts - th Active Users - th Active Content - tr - td #{totalPosts} - - const totalActiveUsers = activeUsers.totalActiveUsers.length > 0 ? activeUsers.totalActiveUsers[0].ips : 0; - td #{totalActiveUsers} - td #{fileStats.totalSizeString} + th Board + th Description + th(title='Posts in the last hour') Posts/h + th(title='Unique IPs that have posted in the last 72h') Active Users + th Total Posts + each board in boards + tr + td: a(href=`/${board._id}/`) /#{board._id}/ - #{board.settings.name} + td #{board.settings.description} + td #{board.pph} + td #{board.ips} + td #{board.sequence_value-1} + .table-container.flex-center.mv-10.text-center + table + tr + th Total Posts + th Active Users + th Active Content + tr + td #{totalPosts} + - const totalActiveUsers = activeUsers.totalActiveUsers.length > 0 ? activeUsers.totalActiveUsers[0].ips : 0; + td #{totalActiveUsers} + td #{fileStats.totalSizeString} .table-container.flex-center.mv-10 table tr @@ -57,3 +58,22 @@ block content | The source code for this site is available a(href='https://github.com/fatchan/jschan/') here | (and in the footer of each page) and is licensed under the Affero General Public License v3. + if webringBoards && webringBoards.length > 0 + .table-container.flex-center.mv-10.text-center + div.scrolltable + table.boardtable.webringtable + tr + th Board + th Description + th(title='Posts in the last hour') Posts/h + th(title='Unique IPs that have posted in the last 72h') Active Users + th Total Posts + each board in webringBoards + tr + td: a(href=board.path) #{board.siteName} /#{board.uri}/ - #{board.title} + td #{board.subtitle || '-'} + td #{board.postsPerHour || '-'} + td #{board.uniqueUsers || '-'} + td #{board.totalPosts || '-'} + p + small: a(href='https://gitlab.com/alogware/LynxChanAddon-Webring') webring? diff --git a/webring.js b/webring.js new file mode 100644 index 00000000..f28e9329 --- /dev/null +++ b/webring.js @@ -0,0 +1,60 @@ +'use strict'; + +const fetch = require('node-fetch') + , { meta } = require(__dirname+'/configs/main.json') + , { following, blacklist } = require(__dirname+'/configs/webring.json') + , { Boards } = require(__dirname+'/db/') + , { outputFile } = require('fs-extra') + , cache = require(__dirname+'/redis.js') + , uploadDirectory = require(__dirname+'/helpers/files/uploadDirectory.js'); + +module.exports = async () => { + //fetch stuff from others + const fetchWebring = [...new Set((await cache.get('webring:sites') || []).concat(following))] + let rings = await Promise.all(fetchWebring.map(url => { + return fetch(url).then(res => res.json()); + })); + let found = []; + let webringBoards = []; + for (let i = 0; i < rings.length; i++) { + //this could really use some validation/sanity checking + const ring = rings[i]; + if (ring.following && ring.following.length > 0) { + found = found.concat(ring.following); + } + if (ring.known && ring.known.length > 0) { + found = found.concat(ring.known); + } + if (ring.boards && ring.boards.length > 0) { + ring.boards.forEach(board => board.siteName = ring.name); + webringBoards = webringBoards.concat(ring.boards); + } + } + const known = [...new Set(found.concat(fetchWebring))] + .filter(site => !blacklist.some(x => site.includes(x))); + //add the known sites and boards to cache in redis (so can be used later in other places e.g. board list) + cache.set('webring:sites', known); + cache.set('webring:boards', webringBoards); + //now update the webring json with board list and known sites + const boards = await Boards.boardSort(0, 0); //does not include unlisted boards + const json = { + name: meta.siteName, + url: meta.url, + endpoint: `${meta.url}/webring.json`, + following, + blacklist, + known, + boards: boards.map(b => { + return { + uri: b._id, + title: b.settings.name, + subtitle: b.settings.description, + path: `${meta.url}/${b._id}/`, + postsPerHour: b.pph, + totalPosts: b.sequence_value-1, + uniqueUsers: b.ips, + }; + }), + } + await outputFile(`${uploadDirectory}json/webring.json`, JSON.stringify(json)); +}