From 014fb2dcad9a5ce2b0dbaecf1b2d76a60439e1e7 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Sat, 24 Jul 2021 17:25:21 +0900 Subject: [PATCH] ExtractorChrome: have frontier consider browser-fetched uris included Since we now run extractors on subresources there's no reason to schedule and refetch them again. Note that duplicate fetches can still occur if the URI was already scheduled or if the browser itself refetches the resource. --- .../java/org/archive/modules/extractor/ExtractorChrome.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index cefa49fc..4d7c3a54 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -275,6 +275,12 @@ public class ExtractorChrome extends ContentExtractor { Frontier frontier = controller.getFrontier(); curi.getOverlayNames(); // for side-effect of creating the overlayNames list + // inform the frontier we've already seen this uri so it won't schedule it + // we only do this for GETs so a POST doesn't prevent scheduling a GET of the same URI + if (request.getMethod().equals("GET")) { + frontier.considerIncluded(curi); + } + KeyedProperties.loadOverridesFrom(curi); try { // perform link extraction