lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 05:35:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Hook in submitted seeds properly.

Browse Source
This commit is contained in:
Andrew Jackson
2015-11-27 20:51:35 +00:00
parent 89cd085e76
commit 0f6d385b24
1 changed files with 30 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+30 -4
View File
@@ -37,6 +37,7 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.SchedulingConstants;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.seeds.SeedModule;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.json.JSONArray;
@@ -77,6 +78,17 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
this.frontier = frontier;
}
protected SeedModule seeds;
public SeedModule getSeeds() {
return this.seeds;
}
@Autowired
public void setSeeds(SeedModule seeds) {
this.seeds = seeds;
}
protected String amqpUri = "amqp://guest:guest@localhost:5672/%2f";
public String getAmqpUri() {
return this.amqpUri;
@@ -303,8 +315,13 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
CrawlURI curi;
try {
curi = makeCrawlUri(jo);
// bypasses scoping (unless rechecking is configured)
getFrontier().schedule(curi);
// Declare as a new seed if it is the case:
if (curi.isSeed()) {
getSeeds().addSeed(curi); // Also triggers scheduling.
} else {
// bypasses scoping (unless rechecking is configured)
getFrontier().schedule(curi);
}
if (logger.isLoggable(Level.FINE)) {
logger.fine("scheduled " + curi);
}
@@ -399,15 +416,24 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
// optional forceFetch instruction:
if (jo.has("forceFetch")) {
boolean forceFetch = jo.getBoolean("forceFetch");
logger.info("Setting forceFetch=" + forceFetch);
if (forceFetch)
logger.info("Setting forceFetch=" + forceFetch);
curi.setForceFetch(forceFetch);
}
// optional isSeed instruction:
if (jo.has("isSeed")) {
boolean isSeed = jo.getBoolean("isSeed");
logger.info("Setting isSeed=" + isSeed);
if (isSeed)
logger.info("Setting isSeed=" + isSeed);
curi.setSeed(isSeed);
// We want to force the seed version of a URL to be fetched even
// if the URL has been seen before. See
//
// org.archive.crawler.postprocessor.CandidatesProcessor.runCandidateChain(CrawlURI
// candidate, CrawlURI source)
//
curi.setForceFetch(true);
}
curi.getAnnotations().add(A_RECEIVED_FROM_AMQP);