diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java index 283b3a1e..dddd26e3 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java @@ -23,6 +23,7 @@ package org.archive.crawler.postprocessor; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; +import org.apache.commons.httpclient.URIException; import org.archive.crawler.framework.Frontier; import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.crawler.spring.SheetOverlaysManager; @@ -33,6 +34,7 @@ import org.archive.modules.SchedulingConstants; import org.archive.modules.extractor.Hop; import org.archive.modules.seeds.SeedModule; import org.archive.spring.KeyedProperties; +import org.archive.util.ArchiveUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -96,6 +98,22 @@ public class CandidatesProcessor extends Processor { public void setSeedsRedirectNewSeeds(boolean redirect) { kp.put("seedsRedirectNewSeeds",redirect); } + + { + setSeedsRedirectNewSeedsAllowTLDs(true); + } + public boolean getSeedsRedirectNewSeedsAllowTLDs() { + return (Boolean) kp.get("seedsRedirectNewSeedsAllowTLDs"); + } + /** + * If enabled, any URL found because a seed redirected to it (original seed + * returned 301 or 302), will also be treated as a seed, as long as the hop + * count is less than {@value #SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS}. + */ + public void setSeedsRedirectNewSeedsAllowTLDs(boolean allowTLDs) { + kp.put("seedsRedirectNewSeedsAllowTLDs",allowTLDs); + } + protected static final int SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS = 5; { @@ -169,7 +187,9 @@ public class CandidatesProcessor extends Processor { // apply special seed-status promotion if(getSeedsRedirectNewSeeds() && source != null && source.isSeed() && candidate.getLastHop().equals(Hop.REFER.getHopString()) - && candidate.getHopCount() < SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS) { + && candidate.getHopCount() < SEEDS_REDIRECT_NEW_SEEDS_MAX_HOPS + && (getSeedsRedirectNewSeedsAllowTLDs() || domainIsNotTLD(candidate)) + ) { candidate.setSeed(true); } @@ -197,6 +217,15 @@ public class CandidatesProcessor extends Processor { } } + private boolean domainIsNotTLD(CrawlURI candidate) { + try { + return !ArchiveUtils.isTld(candidate.getBaseURI().getHost()); + } catch (URIException e) { + // Just swallow this? + } + return false; + } + /** * Run candidates chain on each of (1) any prerequisite, if present; * (2) any outCandidates, if present; (3) all outlinks, if appropriate