mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
Behaviors: log unparseable links through the Page instead of a constructor argument
ExtractLinksBehavior and PaginationBehavior took the UriErrorLoggerModule as a constructor argument, so they had to be Spring beans to get it. That meant the Groovy config couldn't create them inline in the behaviors list, as Spring's Groovy reader doesn't turn inner beans in a list into beans, and had to declare each one as a named bean and refer to it. BrowserProcessor already has the logger module, so it now passes it to behaviors through the page and they can be plain objects.
This commit is contained in:
1 parent
14011df45d
commit
fb3452a6af
5 files changed
+21
-18
No files matched your search
@@ -43,6 +43,7 @@ import org.archive.modules.fetcher.FetchStatusCodes;
|
||||
import org.archive.net.MitmProxy;
|
||||
import org.archive.net.webdriver.*;
|
||||
import org.archive.spring.KeyedProperties;
|
||||
import org.archive.url.URIException;
|
||||
import org.archive.util.IdleBarrier;
|
||||
import org.archive.util.Recorder;
|
||||
import org.eclipse.jetty.client.ProxyConfiguration;
|
||||
@@ -106,6 +107,7 @@ public class BrowserProcessor extends Processor {
|
||||
protected final CrawlController crawlController;
|
||||
protected MitmProxy proxy;
|
||||
protected final FetchHTTP2 fetcher;
|
||||
protected final UriErrorLoggerModule uriErrorLoggerModule;
|
||||
protected final List<Browser> pool = new CopyOnWriteArrayList<>();
|
||||
protected ScheduledExecutorService launcher;
|
||||
protected final Object tabsLock = new Object();
|
||||
@@ -127,7 +129,8 @@ public class BrowserProcessor extends Processor {
|
||||
this.crawlController = crawlController;
|
||||
this.fetcher = fetcher;
|
||||
this.eventPublisher = eventPublisher;
|
||||
this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior(uriErrorLoggerModule));
|
||||
this.uriErrorLoggerModule = uriErrorLoggerModule;
|
||||
this.behaviors = List.of(new ScrollDownBehavior(), new ExtractLinksBehavior());
|
||||
}
|
||||
|
||||
public synchronized void stop() {
|
||||
@@ -269,7 +272,7 @@ public class BrowserProcessor extends Processor {
|
||||
String userAgent = curi.getUserAgent();
|
||||
if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent();
|
||||
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), driver, tab, fetcher.getProxy(), userAgent,
|
||||
new CompletableFuture<>());
|
||||
new CompletableFuture<>(), uriErrorLoggerModule);
|
||||
pages.put(pageId, page);
|
||||
pageIdsByContext.put(tab, pageId);
|
||||
driver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab),
|
||||
@@ -868,7 +871,8 @@ public class BrowserProcessor extends Processor {
|
||||
BrowsingContext.Context context,
|
||||
ProxyConfiguration.Proxy proxy,
|
||||
String userAgent,
|
||||
CompletableFuture<Void> downloadStarted
|
||||
CompletableFuture<Void> downloadStarted,
|
||||
UriErrorLoggerModule uriErrorLoggerModule
|
||||
) implements Page {
|
||||
|
||||
/**
|
||||
@@ -902,5 +906,10 @@ public class BrowserProcessor extends Processor {
|
||||
public <T> T evalPromise(String script, Object... args) {
|
||||
return callFunction(script, true, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void logUriError(URIException e, String link) {
|
||||
if (uriErrorLoggerModule != null) uriErrorLoggerModule.logUriError(e, curi.getUURI(), link);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -86,7 +86,7 @@ class BrowserProcessorTest {
|
||||
@Test
|
||||
public void testPaginationBehavior() throws IOException, InterruptedException {
|
||||
var defaultBehaviors = browserProcessor.getBehaviors();
|
||||
browserProcessor.setBehaviors(List.of(new PaginationBehavior(null)));
|
||||
browserProcessor.setBehaviors(List.of(new PaginationBehavior()));
|
||||
try {
|
||||
CrawlURI crawlURI = newCrawlURI(baseUrl + "paginated");
|
||||
fetcher.process(crawlURI);
|
||||
|
||||
@@ -23,7 +23,6 @@ import org.archive.url.URIException;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
import org.archive.modules.extractor.LinkContext;
|
||||
import org.archive.modules.extractor.UriErrorLoggerModule;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
|
||||
@@ -35,11 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
*/
|
||||
public class ExtractLinksBehavior implements Behavior {
|
||||
private final AtomicLong numberOfLinksExtracted = new AtomicLong(0);
|
||||
private final UriErrorLoggerModule loggerModule;
|
||||
|
||||
public ExtractLinksBehavior(UriErrorLoggerModule loggerModule) {
|
||||
this.loggerModule = loggerModule;
|
||||
}
|
||||
|
||||
public void run(Page page) {
|
||||
List<String> urls = page.eval(/* language=JavaScript */ """
|
||||
@@ -63,7 +57,7 @@ public class ExtractLinksBehavior implements Behavior {
|
||||
page.curi().getOutLinks().add(link);
|
||||
numberOfLinksExtracted.incrementAndGet();
|
||||
} catch (URIException e) {
|
||||
loggerModule.logUriError(e, page.curi().getUURI(), url);
|
||||
page.logUriError(e, url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
package org.archive.modules.behaviors;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.url.URIException;
|
||||
|
||||
/**
|
||||
* Represents a {@link CrawlURI} loaded as web page that a @{@link Behavior} can
|
||||
@@ -31,4 +32,9 @@ public interface Page {
|
||||
<T> T eval(String script, Object... args);
|
||||
|
||||
<T> T evalPromise(String script, Object... args);
|
||||
|
||||
/**
|
||||
* Logs a link found on the page that couldn't be parsed.
|
||||
*/
|
||||
void logUriError(URIException e, String link);
|
||||
}
|
||||
@@ -19,8 +19,6 @@
|
||||
|
||||
package org.archive.modules.behaviors;
|
||||
|
||||
import org.archive.modules.extractor.UriErrorLoggerModule;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
@@ -28,16 +26,12 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
* extracting links after each click. Use as a replacement for ExtractLinksBehavior.
|
||||
*/
|
||||
public class PaginationBehavior implements Behavior {
|
||||
private final ExtractLinksBehavior extractLinks;
|
||||
private final ExtractLinksBehavior extractLinks = new ExtractLinksBehavior();
|
||||
private final AtomicLong pagesClicked = new AtomicLong();
|
||||
private String selector = "button[title=\"Next\"]";
|
||||
private int maxPages = 100;
|
||||
private long loadTimeout = 5000;
|
||||
|
||||
public PaginationBehavior(UriErrorLoggerModule loggerModule) {
|
||||
this.extractLinks = new ExtractLinksBehavior(loggerModule);
|
||||
}
|
||||
|
||||
/**
|
||||
* CSS selector for the next-page button.
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user