From fb3452a6afeb3bdec3dceb636d39ef3b2611f77d Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 13 Nov 2026 16:05:59 +0900 Subject: [PATCH] Behaviors: log unparseable links through the Page instead of a constructor argument ExtractLinksBehavior and PaginationBehavior took the UriErrorLoggerModule as a constructor argument, so they had to be Spring beans to get it. That meant the Groovy config couldn't create them inline in the behaviors list, as Spring's Groovy reader doesn't turn inner beans in a list into beans, and had to declare each one as a named bean and refer to it. BrowserProcessor already has the logger module, so it now passes it to behaviors through the page and they can be plain objects. --- .../crawler/processor/BrowserProcessor.java | 15 ++++++++++++--- .../crawler/processor/BrowserProcessorTest.java | 2 +- .../modules/behaviors/ExtractLinksBehavior.java | 8 +------- .../java/org/archive/modules/behaviors/Page.java | 6 ++++++ .../modules/behaviors/PaginationBehavior.java | 8 +------- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index a99a11b7..c70f7d44 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -43,6 +43,7 @@ import org.archive.modules.fetcher.FetchStatusCodes; import org.archive.net.MitmProxy; import org.archive.net.webdriver.*; import org.archive.spring.KeyedProperties; +import org.archive.url.URIException; import org.archive.util.IdleBarrier; import org.archive.util.Recorder; import org.eclipse.jetty.client.ProxyConfiguration; @@ -106,6 +107,7 @@ public class BrowserProcessor extends Processor { protected final CrawlController crawlController; protected MitmProxy proxy; protected final FetchHTTP2 fetcher; + protected final UriErrorLoggerModule uriErrorLoggerModule; protected final List pool = new CopyOnWriteArrayList<>(); protected ScheduledExecutorService launcher; protected final Object tabsLock = new Object(); @@ -127,7 +129,8 @@ public class BrowserProcessor extends Processor { this.crawlController = crawlController; this.fetcher = fetcher; this.eventPublisher = eventPublisher; - this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior(uriErrorLoggerModule)); + this.uriErrorLoggerModule = uriErrorLoggerModule; + this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior()); } public synchronized void stop() { @@ -269,7 +272,7 @@ public class BrowserProcessor extends Processor { String userAgent = curi.getUserAgent(); if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent(); BrowserPage page = new BrowserPage(curi, new IdleBarrier(), driver, tab, fetcher.getProxy(), userAgent, - new CompletableFuture<>()); + new CompletableFuture<>(), uriErrorLoggerModule); pages.put(pageId, page); pageIdsByContext.put(tab, pageId); driver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab), @@ -868,7 +871,8 @@ public class BrowserProcessor extends Processor { BrowsingContext.Context context, ProxyConfiguration.Proxy proxy, String userAgent, - CompletableFuture downloadStarted + CompletableFuture downloadStarted, + UriErrorLoggerModule uriErrorLoggerModule ) implements Page { /** @@ -902,5 +906,10 @@ public class BrowserProcessor extends Processor { public T evalPromise(String script, Object... args) { return callFunction(script, true, args); } + + @Override + public void logUriError(URIException e, String link) { + if (uriErrorLoggerModule != null) uriErrorLoggerModule.logUriError(e, curi.getUURI(), link); + } } } \ No newline at end of file diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index b5c0e2c9..0b7d82f4 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -86,7 +86,7 @@ class BrowserProcessorTest { @Test public void testPaginationBehavior() throws IOException, InterruptedException { var defaultBehaviors = browserProcessor.getBehaviors(); - browserProcessor.setBehaviors(List.of(new PaginationBehavior(null))); + browserProcessor.setBehaviors(List.of(new PaginationBehavior())); try { CrawlURI crawlURI = newCrawlURI(baseUrl + "paginated"); fetcher.process(crawlURI); diff --git a/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java b/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java index 7b74cc81..0a21f549 100644 --- a/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java +++ b/modules/src/main/java/org/archive/modules/behaviors/ExtractLinksBehavior.java @@ -23,7 +23,6 @@ import org.archive.url.URIException; import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; import org.archive.modules.extractor.LinkContext; -import org.archive.modules.extractor.UriErrorLoggerModule; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -35,11 +34,6 @@ import java.util.concurrent.atomic.AtomicLong; */ public class ExtractLinksBehavior implements Behavior { private final AtomicLong numberOfLinksExtracted = new AtomicLong(0); - private final UriErrorLoggerModule loggerModule; - - public ExtractLinksBehavior(UriErrorLoggerModule loggerModule) { - this.loggerModule = loggerModule; - } public void run(Page page) { List urls = page.eval(/* language=JavaScript */ """ @@ -63,7 +57,7 @@ public class ExtractLinksBehavior implements Behavior { page.curi().getOutLinks().add(link); numberOfLinksExtracted.incrementAndGet(); } catch (URIException e) { - loggerModule.logUriError(e, page.curi().getUURI(), url); + page.logUriError(e, url); } } } diff --git a/modules/src/main/java/org/archive/modules/behaviors/Page.java b/modules/src/main/java/org/archive/modules/behaviors/Page.java index e0415bd4..ce4fa2be 100644 --- a/modules/src/main/java/org/archive/modules/behaviors/Page.java +++ b/modules/src/main/java/org/archive/modules/behaviors/Page.java @@ -20,6 +20,7 @@ package org.archive.modules.behaviors; import org.archive.modules.CrawlURI; +import org.archive.url.URIException; /** * Represents a {@link CrawlURI} loaded as web page that a @{@link Behavior} can @@ -31,4 +32,9 @@ public interface Page { T eval(String script, Object... args); T evalPromise(String script, Object... args); + + /** + * Logs a link found on the page that couldn't be parsed. + */ + void logUriError(URIException e, String link); } diff --git a/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java b/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java index 4bffe9fa..38c0ba43 100644 --- a/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java +++ b/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java @@ -19,8 +19,6 @@ package org.archive.modules.behaviors; -import org.archive.modules.extractor.UriErrorLoggerModule; - import java.util.concurrent.atomic.AtomicLong; /** @@ -28,16 +26,12 @@ import java.util.concurrent.atomic.AtomicLong; * extracting links after each click. Use as a replacement for ExtractLinksBehavior. */ public class PaginationBehavior implements Behavior { - private final ExtractLinksBehavior extractLinks; + private final ExtractLinksBehavior extractLinks = new ExtractLinksBehavior(); private final AtomicLong pagesClicked = new AtomicLong(); private String selector = "button[title=\"Next\"]"; private int maxPages = 100; private long loadTimeout = 5000; - public PaginationBehavior(UriErrorLoggerModule loggerModule) { - this.extractLinks = new ExtractLinksBehavior(loggerModule); - } - /** * CSS selector for the next-page button. */