mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 07:24:41 +00:00
Part of [HER-1836] let more than one extractor run on a single url - new
setting "independentExtractors" - when enabled, extractors run regardless of
whether other extractors have run
* profile-crawler-beans.cxml, AbstractFrontier.java, ExtractorParameters.java,
Extractor.java
new setting independentExtractors
* ContentExtractor.java
shouldProcess() - respect independentExtractors
This commit is contained in:
1 parent
dc056d73f4
commit
acedca1788
5 files changed
+36
-13
No files matched your search
@@ -145,6 +145,15 @@ public abstract class AbstractFrontier
|
||||
kp.put("maxOutlinks", max);
|
||||
}
|
||||
|
||||
{
|
||||
setIndependentExtractors(false);
|
||||
}
|
||||
public boolean isIndependentExtractors() {
|
||||
return (Boolean) kp.get("independentExtractors");
|
||||
}
|
||||
public void setIndependentExtractors(boolean enabled) {
|
||||
kp.put("independentExtractors", enabled);
|
||||
}
|
||||
|
||||
/** inbound updates: URIs to be scheduled, finished; requested state changes */
|
||||
protected BlockingQueue<InEvent> inbound;
|
||||
|
||||
Reference in new issue
Block a user