diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 14e4fab7..d81a1266 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -802,17 +802,11 @@ implements MultiReporter, Serializable, OverlayContext { * If true then a link extractor has already claimed this CrawlURI and * performed link extraction on the document content. This does not * preclude other link extractors that may have an interest in this - * CrawlURI from also doing link extraction but default behavior should - * be to not run if link extraction has already been done. + * CrawlURI from also doing link extraction. * *
There is an onus on link extractors to set this flag if they have * run. * - *
The only extractor of the default Heritrix set that does not - * respect this flag is - * {@link org.archive.crawler.extractor.ExtractorHTTP}. - * It runs against HTTP headers, not the document content. - * * @return True if a processor has performed link extraction on this * CrawlURI * diff --git a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java index 31720578..f5d5effc 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java @@ -42,17 +42,14 @@ public abstract class ContentExtractor extends Extractor { /** * Determines if links should be extracted from the given URI. This - * method performs three checks. The first is to check the URI's - * {@link ExtractorURI#hasBeenLinkExtracted()} result. If that - * result is true, then this method returns false, as some other - * extractor has claimed that links are already extracted. + * method performs two checks. * - *
Next, this method checks that the content length of the URI is + *
First, this method checks that the content length of the URI is * greater than zero (in other words, that there is actually content * for links to be extracted from). If the content length of the URI * is zero or less, then this method returns false. * - *
Finally, this method delegates to {@link #innerExtract(ExtractorURI)} + *
Finally, this method delegates to {@link #shouldExtract(CrawlURI))} * and returns that result. * * @param uri the URI to check @@ -60,9 +57,6 @@ public abstract class ContentExtractor extends Extractor { * false otherwise */ final protected boolean shouldProcess(CrawlURI uri) { - if (uri.hasBeenLinkExtracted()) { - return false; - } if (uri.getContentLength() <= 0) { return false; } @@ -72,19 +66,16 @@ public abstract class ContentExtractor extends Extractor { return true; } - /** * Determines if otherwise valid URIs should have links extracted or not. - * The given URI will not have its - * {@link ExtractorURI#hasBeenLinkExtracted()} flag set, and its - * content length will be greater than zero. Subclasses should - * implement this method to perform additional checks. For instance, - * the {@link ExtractorHTML} implementation checks that the content-type - * of the given URI is text/html. + * The given URI will have content length greater than zero. Subclasses + * should implement this method to perform additional checks. For instance, + * the {@link ExtractorHTML} implementation checks that the content-type of + * the given URI is text/html. * - * @param uri the URI to check - * @return true if links should be extracted from that URI, false - * otherwise + * @param uri + * the URI to check + * @return true if links should be extracted from that URI, false otherwise */ protected abstract boolean shouldExtract(CrawlURI uri);