lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 14:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1836] let more than one extractor run on a single url?

Browse Source 
* ContentExtractor.java
    shouldProcess() - do not check uri.hasBeenLinkExtracted()
* CrawlURI.java
    update javadoc comments on hasLinkBeenExtracted()
This commit is contained in:
nlevitt
2010-10-19 02:08:54 +00:00
parent d72e6cd1d0
commit b75f604bf9
2 changed files with 11 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+1 -7
View File
@@ -802,17 +802,11 @@ implements MultiReporter, Serializable, OverlayContext {
* If true then a link extractor has already claimed this CrawlURI and
* performed link extraction on the document content. This does not
* preclude other link extractors that may have an interest in this
* CrawlURI from also doing link extraction but default behavior should
* be to not run if link extraction has already been done.
* CrawlURI from also doing link extraction.
*
* <p>There is an onus on link extractors to set this flag if they have
* run.
*
* <p>The only extractor of the default Heritrix set that does not
* respect this flag is
* {@link org.archive.crawler.extractor.ExtractorHTTP}.
* It runs against HTTP headers, not the document content.
*
* @return True if a processor has performed link extraction on this
* CrawlURI
*
modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java
+10 -19
View File
@@ -42,17 +42,14 @@ public abstract class ContentExtractor extends Extractor {
/**
* Determines if links should be extracted from the given URI. This
* method performs three checks. The first is to check the URI's
* {@link ExtractorURI#hasBeenLinkExtracted()} result. If that
* result is true, then this method returns false, as some other
* extractor has claimed that links are already extracted.
* method performs two checks.
*
* <p>Next, this method checks that the content length of the URI is
* <p>First, this method checks that the content length of the URI is
* greater than zero (in other words, that there is actually content
* for links to be extracted from). If the content length of the URI
* is zero or less, then this method returns false.
*
* <p>Finally, this method delegates to {@link #innerExtract(ExtractorURI)}
* <p>Finally, this method delegates to {@link #shouldExtract(CrawlURI))}
* and returns that result.
*
* @param uri the URI to check
@@ -60,9 +57,6 @@ public abstract class ContentExtractor extends Extractor {
* false otherwise
*/
final protected boolean shouldProcess(CrawlURI uri) {
if (uri.hasBeenLinkExtracted()) {
return false;
}
if (uri.getContentLength() <= 0) {
return false;
}
@@ -72,19 +66,16 @@ public abstract class ContentExtractor extends Extractor {
return true;
}
/**
* Determines if otherwise valid URIs should have links extracted or not.
* The given URI will not have its
* {@link ExtractorURI#hasBeenLinkExtracted()} flag set, and its
* content length will be greater than zero. Subclasses should
* implement this method to perform additional checks. For instance,
* the {@link ExtractorHTML} implementation checks that the content-type
* of the given URI is text/html.
* The given URI will have content length greater than zero. Subclasses
* should implement this method to perform additional checks. For instance,
* the {@link ExtractorHTML} implementation checks that the content-type of
* the given URI is text/html.
*
* @param uri the URI to check
* @return true if links should be extracted from that URI, false
* otherwise
* @param uri
* the URI to check
* @return true if links should be extracted from that URI, false otherwise
*/
protected abstract boolean shouldExtract(CrawlURI uri);