diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index a99a11b7..c70f7d44 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -43,6 +43,7 @@ import org.archive.modules.fetcher.FetchStatusCodes; import org.archive.net.MitmProxy; import org.archive.net.webdriver.*; import org.archive.spring.KeyedProperties; +import org.archive.url.URIException; import org.archive.util.IdleBarrier; import org.archive.util.Recorder; import org.eclipse.jetty.client.ProxyConfiguration; @@ -106,6 +107,7 @@ public class BrowserProcessor extends Processor { protected final CrawlController crawlController; protected MitmProxy proxy; protected final FetchHTTP2 fetcher; + protected final UriErrorLoggerModule uriErrorLoggerModule; protected final List pool = new CopyOnWriteArrayList<>(); protected ScheduledExecutorService launcher; protected final Object tabsLock = new Object(); @@ -127,7 +129,8 @@ public class BrowserProcessor extends Processor { this.crawlController = crawlController; this.fetcher = fetcher; this.eventPublisher = eventPublisher; - this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior(uriErrorLoggerModule)); + this.uriErrorLoggerModule = uriErrorLoggerModule; + this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior()); } public synchronized void stop() { @@ -269,7 +272,7 @@ public class BrowserProcessor extends Processor { String userAgent = curi.getUserAgent(); if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent(); BrowserPage page = new BrowserPage(curi, new IdleBarrier(), driver, tab, fetcher.getProxy(), userAgent, - new CompletableFuture<>()); + new CompletableFuture<>(), uriErrorLoggerModule); pages.put(pageId, page); pageIdsByContext.put(tab, pageId); driver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab), @@ -868,7 +871,8 @@ public class BrowserProcessor extends Processor { BrowsingContext.Context context, ProxyConfiguration.Proxy proxy, String userAgent, - CompletableFuture downloadStarted + CompletableFuture downloadStarted, + UriErrorLoggerModule uriErrorLoggerModule ) implements Page { /** @@ -902,5 +906,10 @@ public class BrowserProcessor extends Processor { public T evalPromise(String script, Object... args) { return callFunction(script, true, args); } + + @Override + public void logUriError(URIException e, String link) { + if (uriErrorLoggerModule != null) uriErrorLoggerModule.logUriError(e, curi.getUURI(), link); + } } } \ No newline at end of file diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index b5c0e2c9..0b7d82f4 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -86,7 +86,7 @@ class BrowserProcessorTest { @Test public void testPaginationBehavior() throws IOException, InterruptedException { var defaultBehaviors = browserProcessor.getBehaviors(); - browserProcessor.setBehaviors(List.of(new PaginationBehavior(null))); + browserProcessor.setBehaviors(List.of(new PaginationBehavior())); try { CrawlURI crawlURI = newCrawlURI(baseUrl + "paginated"); fetcher.process(crawlURI); diff --git a/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java b/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java index 7b74cc81..0a21f549 100644 --- a/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java +++ b/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java @@ -23,7 +23,6 @@ import org.archive.url.URIException; import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; import org.archive.modules.extractor.LinkContext; -import org.archive.modules.extractor.UriErrorLoggerModule; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -35,11 +34,6 @@ import java.util.concurrent.atomic.AtomicLong; */ public class ExtractLinksBehavior implements Behavior { private final AtomicLong numberOfLinksExtracted = new AtomicLong(0); - private final UriErrorLoggerModule loggerModule; - - public ExtractLinksBehavior(UriErrorLoggerModule loggerModule) { - this.loggerModule = loggerModule; - } public void run(Page page) { List urls = page.eval(/* language=JavaScript */ """ @@ -63,7 +57,7 @@ public class ExtractLinksBehavior implements Behavior { page.curi().getOutLinks().add(link); numberOfLinksExtracted.incrementAndGet(); } catch (URIException e) { - loggerModule.logUriError(e, page.curi().getUURI(), url); + page.logUriError(e, url); } } } diff --git a/modules/src/main/java/org/archive/modules/behaviors/Page.java b/modules/src/main/java/org/archive/modules/behaviors/Page.java index e0415bd4..ce4fa2be 100644 --- a/modules/src/main/java/org/archive/modules/behaviors/Page.java +++ b/modules/src/main/java/org/archive/modules/behaviors/Page.java @@ -20,6 +20,7 @@ package org.archive.modules.behaviors; import org.archive.modules.CrawlURI; +import org.archive.url.URIException; /** * Represents a {@link CrawlURI} loaded as web page that a @{@link Behavior} can @@ -31,4 +32,9 @@ public interface Page { T eval(String script, Object... args); T evalPromise(String script, Object... args); + + /** + * Logs a link found on the page that couldn't be parsed. + */ + void logUriError(URIException e, String link); } diff --git a/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java b/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java index 4bffe9fa..38c0ba43 100644 --- a/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java +++ b/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java @@ -19,8 +19,6 @@ package org.archive.modules.behaviors; -import org.archive.modules.extractor.UriErrorLoggerModule; - import java.util.concurrent.atomic.AtomicLong; /** @@ -28,16 +26,12 @@ import java.util.concurrent.atomic.AtomicLong; * extracting links after each click. Use as a replacement for ExtractLinksBehavior. */ public class PaginationBehavior implements Behavior { - private final ExtractLinksBehavior extractLinks; + private final ExtractLinksBehavior extractLinks = new ExtractLinksBehavior(); private final AtomicLong pagesClicked = new AtomicLong(); private String selector = "button[title=\"Next\"]"; private int maxPages = 100; private long loadTimeout = 5000; - public PaginationBehavior(UriErrorLoggerModule loggerModule) { - this.extractLinks = new ExtractLinksBehavior(loggerModule); - } - /** * CSS selector for the next-page button. */