lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #461 from kris-sigur/seed-redirect-tld

Browse Source 
Add conf to not allow TLDs as seeds found via redirect from other seeds
This commit is contained in:
Alex Osborne
2022-02-14 07:46:09 +09:00
committed by GitHub
parent 4a75fe1fea 69b023199d
commit 568d5cedfe
1 changed files with 30 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
+30 -1
View File
@@ -23,6 +23,7 @@ package org.archive.crawler.postprocessor;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
import org.apache.commons.httpclient.URIException;
import org.archive.crawler.framework.Frontier;
import org.archive.crawler.reporting.CrawlerLoggerModule;
import org.archive.crawler.spring.SheetOverlaysManager;
@@ -33,6 +34,7 @@ import org.archive.modules.SchedulingConstants;
import org.archive.modules.extractor.Hop;
import org.archive.modules.seeds.SeedModule;
import org.archive.spring.KeyedProperties;
import org.archive.util.ArchiveUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -96,6 +98,22 @@ public class CandidatesProcessor extends Processor {
public void setSeedsRedirectNewSeeds(boolean redirect) {
kp.put("seedsRedirectNewSeeds",redirect);
}
{
setSeedsRedirectNewSeedsAllowTLDs(true);
}
public boolean getSeedsRedirectNewSeedsAllowTLDs() {
return (Boolean) kp.get("seedsRedirectNewSeedsAllowTLDs");
}
/**
* If enabled, any URL found because a seed redirected to it (original seed
* returned 301 or 302), will also be treated as a seed, as long as the hop
* count is less than {@value #SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS}.
*/
public void setSeedsRedirectNewSeedsAllowTLDs(boolean allowTLDs) {
kp.put("seedsRedirectNewSeedsAllowTLDs",allowTLDs);
}
protected static final int SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS = 5;
{
@@ -169,7 +187,9 @@ public class CandidatesProcessor extends Processor {
// apply special seed-status promotion
if(getSeedsRedirectNewSeeds() && source != null && source.isSeed()
&& candidate.getLastHop().equals(Hop.REFER.getHopString())
&& candidate.getHopCount() < SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS) {
&& candidate.getHopCount() < SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS
&& (getSeedsRedirectNewSeedsAllowTLDs() || domainIsNotTLD(candidate))
) {
candidate.setSeed(true);
}
@@ -197,6 +217,15 @@ public class CandidatesProcessor extends Processor {
}
}
private boolean domainIsNotTLD(CrawlURI candidate) {
try {
return !ArchiveUtils.isTld(candidate.getBaseURI().getHost());
} catch (URIException e) {
// Just swallow this?
}
return false;
}
/**
* Run candidates chain on each of (1) any prerequisite, if present;
* (2) any outCandidates, if present; (3) all outlinks, if appropriate