Merge branch 'develop' into 'master'

0.3.1 webring patches

See merge request fatchan/jschan!238

CHANGELOG.md

@@ -1,3 +1,10 @@
+### 0.3.1
+  - Small board list optimisation, no longer try to fetch webring site names if webring not enabled.
+  - Make webring blacklisting also apply to endpoints after being fetched, in case the URL is different (alt domain, onions, etc).
+  - Webring.json and site names cache is now removed when webring is disabled.
+  - More reasonable webring timeout for fetching each site.
+  - Webring bugfixes.
+
 ### 0.3.0
   - Ban durations are now editable. Staff can set a new ban duration (starting from the current date, not the original ban issue date).
   - Minor bugfixes.

models/forms/changeglobalsettings.js

@@ -324,9 +324,11 @@ module.exports = async (req, res, next) => {
 
 	await Mongo.setConfig(newSettings);
 
+	//webring being disabled
 	if (oldSettings.enableWebring === true && newSettings.enableWebring === false) {
-		//delete webring boards from boardlist when disabling.
-		await Boards.db.deleteMany({ webring: true });
+		promises.push(Boards.db.deleteMany({ webring: true }));
+		promises.push(remove(`${uploadDirectory}/json/webring.json`));
+		redis.del('webringsites');
 	}
 
 	//finish the promises in parallel e.g. removing files

models/pages/boardlist.js

@@ -9,7 +9,7 @@ const config = require(__dirname+'/../../config.js')
 
 module.exports = async (req, res, next) => {
 
-	const { meta } = config.get;
+	const { meta, enableWebring } = config.get;
 	const { page, offset, queryString } = pageQueryConverter(req.query, limit);
 	const direction = req.query.direction && req.query.direction === 'asc' ? 1 : -1;
 	const search = (typeof req.query.search === 'string' ? req.query.search : null);
@@ -17,13 +17,13 @@ module.exports = async (req, res, next) => {
 	const sortType = req.query.sort && req.query.sort === 'activity' ? 'activity' : 'popularity';
 	let sort = {};
 
-	const webringSites = [meta.siteName, ...(await Boards.webringSites())];
-	const webringSitesSet = new Set(webringSites);
+	const siteNames = enableWebring === true ? [meta.siteName, ...(await Boards.webringSites())] : [meta.siteName];
+	const siteNamesSet = new Set(siteNames);
 
 	let selectedSites = req.query.sites ? (typeof req.query.sites === 'string' ? [req.query.sites] : req.query.sites) : [];
-	let validSelectedSites = selectedSites.filter(ss => webringSitesSet.has(ss));
+	let validSelectedSites = selectedSites.filter(ss => siteNamesSet.has(ss));
 	if (validSelectedSites.length === 0) {
-		validSelectedSites = webringSites;
+		validSelectedSites = siteNames;
 	}
 	const validSelectedSitesSet = new Set(validSelectedSites);
 
@@ -31,11 +31,11 @@ module.exports = async (req, res, next) => {
 	cacheQuery.sort();
 	const cacheQueryString = cacheQuery.toString();
 	const cached = await cache.get(`boardlist:${cacheQueryString}`);
-	const { shown, notShown } = webringSites.reduce((acc, ws) => {
-		if (validSelectedSitesSet.has(ws)) {
-			acc.shown.push(ws);
+	const { shown, notShown } = siteNames.reduce((acc, sn) => {
+		if (validSelectedSitesSet.has(sn)) {
+			acc.shown.push(sn);
 		} else {
-			acc.notShown.push(ws)
+			acc.notShown.push(sn);
 		}
 		return acc;
 	}, { shown: [], notShown: [] });

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "jschan",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "jschan",
-      "version": "0.3.0",
+      "version": "0.3.1",
       "license": "AGPL-3.0-only",
       "dependencies": {
         "@fatchan/express-fileupload": "^1.3.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jschan",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "migrateVersion": "0.2.0",
   "description": "",
   "main": "server.js",

schedules/tasks/webring.js

@@ -29,8 +29,9 @@ module.exports = {
 			//get sites we havent visited yet
 			const toVisit = [...known].filter(url => !visited.has(url));
 			let rings = await Promise.all(toVisit.map(url => {
-				visited.set(url, (visited.get(url)||0)+1);
+				visited.set(url, (visited.get(url)||1));
 				return fetch(url, {
+					timeout: 20000,
 					agent,
 					headers: {
 						'User-Agent':''
@@ -43,10 +44,11 @@ module.exports = {
 				const ring = rings[i];
 				if (!ring || !ring.name || !ring.endpoint || !ring.url //malformed
 					|| ring.endpoint.includes(meta.url) //own site
+					|| blacklist.some(x => ring.endpoint.includes(x)) //blacklisted (for the case of a mirror to the endpoint)
 					|| visited.get(ring.endpoint) > 1) { //already seen endpoint (for multiple domain sites)
 					continue;
 				}
-				visited.set(ring.endpoint, visited.get(ring.endpoint)+1);
+				visited.set(ring.endpoint, (visited.get(ring.endpoint)||1)+1);
 				if (ring.following && ring.following.length > 0) {
 					//filter their folowing by blacklist/self and add to known sites
 					ring.following