|
|
@ -29,8 +29,9 @@ module.exports = { |
|
|
|
//get sites we havent visited yet
|
|
|
|
//get sites we havent visited yet
|
|
|
|
const toVisit = [...known].filter(url => !visited.has(url)); |
|
|
|
const toVisit = [...known].filter(url => !visited.has(url)); |
|
|
|
let rings = await Promise.all(toVisit.map(url => { |
|
|
|
let rings = await Promise.all(toVisit.map(url => { |
|
|
|
visited.set(url, (visited.get(url)||0)+1); |
|
|
|
visited.set(url, (visited.get(url)||1)); |
|
|
|
return fetch(url, { |
|
|
|
return fetch(url, { |
|
|
|
|
|
|
|
timeout: 20000, |
|
|
|
agent, |
|
|
|
agent, |
|
|
|
headers: { |
|
|
|
headers: { |
|
|
|
'User-Agent':'' |
|
|
|
'User-Agent':'' |
|
|
@ -47,7 +48,7 @@ module.exports = { |
|
|
|
|| visited.get(ring.endpoint) > 1) { //already seen endpoint (for multiple domain sites)
|
|
|
|
|| visited.get(ring.endpoint) > 1) { //already seen endpoint (for multiple domain sites)
|
|
|
|
continue; |
|
|
|
continue; |
|
|
|
} |
|
|
|
} |
|
|
|
visited.set(ring.endpoint, visited.get(ring.endpoint)+1); |
|
|
|
visited.set(ring.endpoint, (visited.get(ring.endpoint)||1)+1); |
|
|
|
if (ring.following && ring.following.length > 0) { |
|
|
|
if (ring.following && ring.following.length > 0) { |
|
|
|
//filter their folowing by blacklist/self and add to known sites
|
|
|
|
//filter their folowing by blacklist/self and add to known sites
|
|
|
|
ring.following |
|
|
|
ring.following |
|
|
|