diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index cefa49fc..4d7c3a54 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -275,6 +275,12 @@ public class ExtractorChrome extends ContentExtractor { Frontier frontier = controller.getFrontier(); curi.getOverlayNames(); // for side-effect of creating the overlayNames list + // inform the frontier we've already seen this uri so it won't schedule it + // we only do this for GETs so a POST doesn't prevent scheduling a GET of the same URI + if (request.getMethod().equals("GET")) { + frontier.considerIncluded(curi); + } + KeyedProperties.loadOverridesFrom(curi); try { // perform link extraction