set a (still generous) 20s timeout instead of whatever insane node-fetch default is, seems like 60s or 120s

small bugfix to visit counts
indiachan-spamvector
Thomas Lynch 2 years ago
parent cc091a2d4d
commit ef40721ec1
Signed by: fatchan
GPG Key ID: 112884AA57DF40B1
  1. 5
      schedules/tasks/webring.js

@ -29,8 +29,9 @@ module.exports = {
//get sites we havent visited yet //get sites we havent visited yet
const toVisit = [...known].filter(url => !visited.has(url)); const toVisit = [...known].filter(url => !visited.has(url));
let rings = await Promise.all(toVisit.map(url => { let rings = await Promise.all(toVisit.map(url => {
visited.set(url, (visited.get(url)||0)+1); visited.set(url, (visited.get(url)||1));
return fetch(url, { return fetch(url, {
timeout: 20000,
agent, agent,
headers: { headers: {
'User-Agent':'' 'User-Agent':''
@ -47,7 +48,7 @@ module.exports = {
|| visited.get(ring.endpoint) > 1) { //already seen endpoint (for multiple domain sites) || visited.get(ring.endpoint) > 1) { //already seen endpoint (for multiple domain sites)
continue; continue;
} }
visited.set(ring.endpoint, visited.get(ring.endpoint)+1); visited.set(ring.endpoint, (visited.get(ring.endpoint)||1)+1);
if (ring.following && ring.following.length > 0) { if (ring.following && ring.following.length > 0) {
//filter their folowing by blacklist/self and add to known sites //filter their folowing by blacklist/self and add to known sites
ring.following ring.following

Loading…
Cancel
Save