Behaviors: log unparseable links through the Page instead of a constructor argument

ExtractLinksBehavior and PaginationBehavior took the UriErrorLoggerModule as a
constructor argument, so they had to be Spring beans to get it. That meant the
Groovy config couldn't create them inline in the behaviors list, as Spring's
Groovy reader doesn't turn inner beans in a list into beans, and had to declare
each one as a named bean and refer to it. BrowserProcessor already has the
logger module, so it now passes it to behaviors through the page and they can
be plain objects.
This commit is contained in:
Alex Osborne committed 2026-11-13 16:08:36 +09:00
1 parent 14011df45d
commit fb3452a6af
5 files changed
+21 -18

No files matched your search

@@ -43,6 +43,7 @@ import org.archive.modules.fetcher.FetchStatusCodes;
import org.archive.net.MitmProxy;
import org.archive.net.webdriver.*;
import org.archive.spring.KeyedProperties;
import org.archive.url.URIException;
import org.archive.util.IdleBarrier;
import org.archive.util.Recorder;
import org.eclipse.jetty.client.ProxyConfiguration;
@@ -106,6 +107,7 @@ public class BrowserProcessor extends Processor {
protected final CrawlController crawlController;
protected MitmProxy proxy;
protected final FetchHTTP2 fetcher;
protected final UriErrorLoggerModule uriErrorLoggerModule;
protected final List<Browser> pool = new CopyOnWriteArrayList<>();
protected ScheduledExecutorService launcher;
protected final Object tabsLock = new Object();
@@ -127,7 +129,8 @@ public class BrowserProcessor extends Processor {
this.crawlController = crawlController;
this.fetcher = fetcher;
this.eventPublisher = eventPublisher;
this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior(uriErrorLoggerModule));
this.uriErrorLoggerModule = uriErrorLoggerModule;
this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior());
}
public synchronized void stop() {
@@ -269,7 +272,7 @@ public class BrowserProcessor extends Processor {
String userAgent = curi.getUserAgent();
if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent();
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), driver, tab, fetcher.getProxy(), userAgent,
new CompletableFuture<>());
new CompletableFuture<>(), uriErrorLoggerModule);
pages.put(pageId, page);
pageIdsByContext.put(tab, pageId);
driver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab),
@@ -868,7 +871,8 @@ public class BrowserProcessor extends Processor {
BrowsingContext.Context context,
ProxyConfiguration.Proxy proxy,
String userAgent,
CompletableFuture<Void> downloadStarted
CompletableFuture<Void> downloadStarted,
UriErrorLoggerModule uriErrorLoggerModule
) implements Page {
/**
@@ -902,5 +906,10 @@ public class BrowserProcessor extends Processor {
public <T> T evalPromise(String script, Object... args) {
return callFunction(script, true, args);
}
@Override
public void logUriError(URIException e, String link) {
if (uriErrorLoggerModule != null) uriErrorLoggerModule.logUriError(e, curi.getUURI(), link);
}
}
}
@@ -86,7 +86,7 @@ class BrowserProcessorTest {
@Test
public void testPaginationBehavior() throws IOException, InterruptedException {
var defaultBehaviors = browserProcessor.getBehaviors();
browserProcessor.setBehaviors(List.of(new PaginationBehavior(null)));
browserProcessor.setBehaviors(List.of(new PaginationBehavior()));
try {
CrawlURI crawlURI = newCrawlURI(baseUrl + "paginated");
fetcher.process(crawlURI);
@@ -23,7 +23,6 @@ import org.archive.url.URIException;
import org.archive.modules.CrawlURI;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.extractor.UriErrorLoggerModule;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -35,11 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class ExtractLinksBehavior implements Behavior {
private final AtomicLong numberOfLinksExtracted = new AtomicLong(0);
private final UriErrorLoggerModule loggerModule;
public ExtractLinksBehavior(UriErrorLoggerModule loggerModule) {
this.loggerModule = loggerModule;
}
public void run(Page page) {
List<String> urls = page.eval(/* language=JavaScript */ """
@@ -63,7 +57,7 @@ public class ExtractLinksBehavior implements Behavior {
page.curi().getOutLinks().add(link);
numberOfLinksExtracted.incrementAndGet();
} catch (URIException e) {
loggerModule.logUriError(e, page.curi().getUURI(), url);
page.logUriError(e, url);
}
}
}
@@ -20,6 +20,7 @@
package org.archive.modules.behaviors;
import org.archive.modules.CrawlURI;
import org.archive.url.URIException;
/**
* Represents a {@link CrawlURI} loaded as web page that a @{@link Behavior} can
@@ -31,4 +32,9 @@ public interface Page {
<T> T eval(String script, Object... args);
<T> T evalPromise(String script, Object... args);
/**
* Logs a link found on the page that couldn't be parsed.
*/
void logUriError(URIException e, String link);
}
@@ -19,8 +19,6 @@
package org.archive.modules.behaviors;
import org.archive.modules.extractor.UriErrorLoggerModule;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -28,16 +26,12 @@ import java.util.concurrent.atomic.AtomicLong;
* extracting links after each click. Use as a replacement for ExtractLinksBehavior.
*/
public class PaginationBehavior implements Behavior {
private final ExtractLinksBehavior extractLinks;
private final ExtractLinksBehavior extractLinks = new ExtractLinksBehavior();
private final AtomicLong pagesClicked = new AtomicLong();
private String selector = "button[title=\"Next\"]";
private int maxPages = 100;
private long loadTimeout = 5000;
public PaginationBehavior(UriErrorLoggerModule loggerModule) {
this.extractLinks = new ExtractLinksBehavior(loggerModule);
}
/**
* CSS selector for the next-page button.
*/