lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-09 15:36:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Followup for [HER-1605] Refactor parameterizable policies out of frontier

Browse Source 
* CandidatesProcessor.java
    remove misguided lock preventing intended parallelism of pre-enqueue URL prep
This commit is contained in:
gojomo committed 2009-10-21 22:55:06 +00:00
1 parent ea114357fb
commit f0c9608e59
1 file changed
+33 -42
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
+33 -42
View File
@@ -23,8 +23,6 @@ package org.archive.crawler.postprocessor;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.httpclient.URIException;
import org.archive.crawler.framework.Frontier;
import org.archive.crawler.reporting.CrawlerLoggerModule;
@@ -49,8 +47,6 @@ import org.springframework.beans.factory.annotation.Autowired;
*/
public class CandidatesProcessor extends Processor {
private static final long serialVersionUID = -3L;
ReentrantLock lock = new ReentrantLock(true);
/**
* Candidate chain
@@ -163,49 +159,44 @@ public class CandidatesProcessor extends Processor {
curi.getOutLinks().clear();
return;
}
try {
lock.lock();
for (Link wref: curi.getOutLinks()) {
CrawlURI candidate;
try {
candidate = curi.createCrawlURI(curi.getBaseURI(),wref);
// at least for duration of candidatechain, offer
// access to full CrawlURI of via
candidate.setFullVia(curi);
} catch (URIException e) {
loggerModule.logUriError(e, curi.getUURI(),
wref.getDestination().toString());
continue;
for (Link wref: curi.getOutLinks()) {
CrawlURI candidate;
try {
candidate = curi.createCrawlURI(curi.getBaseURI(),wref);
// at least for duration of candidatechain, offer
// access to full CrawlURI of via
candidate.setFullVia(curi);
} catch (URIException e) {
loggerModule.logUriError(e, curi.getUURI(),
wref.getDestination().toString());
continue;
}
sheetOverlaysManager.applyOverridesTo(candidate);
try {
KeyedProperties.clearOverridesFrom(curi);
KeyedProperties.loadOverridesFrom(candidate);
if(getSeedsRedirectNewSeeds() && curi.isSeed()
&& wref.getHopType() == Hop.REFER) {
candidate.setSeed(true);
}
sheetOverlaysManager.applyOverridesTo(candidate);
try {
KeyedProperties.clearOverridesFrom(curi);
KeyedProperties.loadOverridesFrom(candidate);
if(getSeedsRedirectNewSeeds() && curi.isSeed()
&& wref.getHopType() == Hop.REFER) {
candidate.setSeed(true);
getCandidateChain().process(candidate, null);
if(candidate.getFetchStatus()>=0) {
if(checkForSeedPromotion(candidate)) {
getSeeds().addSeed(candidate);
} else {
frontier.schedule(candidate);
}
getCandidateChain().process(candidate, null);
if(candidate.getFetchStatus()>=0) {
if(checkForSeedPromotion(candidate)) {
getSeeds().addSeed(candidate);
} else {
frontier.schedule(candidate);
}
curi.getOutCandidates().add(candidate);
}
} finally {
KeyedProperties.clearOverridesFrom(candidate);
KeyedProperties.loadOverridesFrom(curi);
curi.getOutCandidates().add(candidate);
}
} finally {
KeyedProperties.clearOverridesFrom(candidate);
KeyedProperties.loadOverridesFrom(curi);
}
curi.getOutLinks().clear();
} finally {
lock.unlock();
}
curi.getOutLinks().clear();
}
/**