diff --git a/commons/src/main/java/org/archive/net/webdriver/BiDiJson.java b/commons/src/main/java/org/archive/net/webdriver/BiDiJson.java index b4d43d7c..7b9b6bc0 100644 --- a/commons/src/main/java/org/archive/net/webdriver/BiDiJson.java +++ b/commons/src/main/java/org/archive/net/webdriver/BiDiJson.java @@ -185,10 +185,13 @@ class BiDiJson { } private static Class getSubclassByTypeName(Class type, String typeName) { + Class fallback = null; for (Class subclass : type.getPermittedSubclasses()) { TypeName annotation = subclass.getDeclaredAnnotation(TypeName.class); if (annotation != null && annotation.value().equals(typeName)) return subclass; + if (subclass.getDeclaredAnnotation(UnknownTypeFallback.class) != null) fallback = subclass; } + if (fallback != null) return fallback; throw new UnsupportedOperationException("Unsupported 'type' for " + type + ": " + typeName); } @@ -208,6 +211,14 @@ class BiDiJson { String value(); } + /** + * Marks the subclass to deserialize into when the 'type' field matches no other permitted subclass. + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + @interface UnknownTypeFallback { + } + @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.RECORD_COMPONENT) @interface PropertyName { diff --git a/commons/src/main/java/org/archive/net/webdriver/Script.java b/commons/src/main/java/org/archive/net/webdriver/Script.java index fb332079..167b09ff 100644 --- a/commons/src/main/java/org/archive/net/webdriver/Script.java +++ b/commons/src/main/java/org/archive/net/webdriver/Script.java @@ -58,6 +58,8 @@ public interface Script extends BiDiModule { sealed interface LocalValue { static LocalValue from(Object object) { if (object instanceof Number number) return new NumberValue(number); + if (object instanceof String string) return new StringValue(string); + if (object instanceof Boolean bool) return new BooleanValue(bool); throw new IllegalArgumentException("Unsupported local value type: " + object.getClass()); } } @@ -82,6 +84,34 @@ public interface Script extends BiDiModule { } } + @BiDiJson.TypeName("boolean") + record BooleanValue(boolean value) implements RemoteValue, LocalValue { + @Override + public Boolean javaValue() { + return value; + } + } + + @BiDiJson.TypeName("null") + record NullValue() implements RemoteValue, LocalValue { + @Override + public Object javaValue() { + return null; + } + } + + /** + * Catch-all for RemoteValue types that aren't modelled here, as page scripts can return any + * type the protocol defines. The value is the raw JSON 'value' field, if there was one. + */ + @BiDiJson.UnknownTypeFallback + record UnknownValue(String type, Object value) implements RemoteValue { + @Override + public Object javaValue() { + return value; + } + } + @BiDiJson.TypeName("array") record ArrayRemoteValue(List value) implements RemoteValue, LocalValue { @Override diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index 066831b4..5a58ea34 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -3,6 +3,7 @@ package org.archive.crawler.processor; import com.sun.net.httpserver.HttpServer; import org.archive.crawler.framework.CrawlController; import org.archive.modules.*; +import org.archive.modules.behaviors.PaginationBehavior; import org.archive.modules.fetcher.DefaultServerCache; import org.archive.modules.fetcher.FetchHTTP2; import org.archive.net.UURIFactory; @@ -62,6 +63,27 @@ class BrowserProcessorTest { assertEquals("/*hello world*/", gzip.getRecorder().getContentReplayPrefixString(100)); } + @Test + public void testPaginationBehavior() throws IOException, InterruptedException { + var defaultBehaviors = browserProcessor.getBehaviors(); + browserProcessor.setBehaviors(List.of(new PaginationBehavior(null))); + try { + CrawlURI crawlURI = newCrawlURI(baseUrl + "paginated"); + fetcher.process(crawlURI); + assertEquals(200, crawlURI.getFetchStatus()); + browserProcessor.innerProcess(crawlURI); + + var paths = new HashSet(); + for (CrawlURI link : crawlURI.getOutLinks()) { + paths.add(link.getUURI().getPath()); + } + assertEquals(Set.of("/item1", "/item2", "/item3"), paths, + "should have extracted the links from every page of the pagination"); + } finally { + browserProcessor.setBehaviors(defaultBehaviors); + } + } + @Test public void testDownload() throws IOException, InterruptedException { CrawlURI crawlURI = newCrawlURI(baseUrl + "download.bin"); @@ -143,6 +165,18 @@ class BrowserProcessorTest { body = "body { color: red; background: url(bg.jpg); }"; contentType = "text/css"; } + case "/paginated" -> body = """ + + + """; case "/download.bin" -> { body = "sample-download-file"; exchange.getResponseHeaders().add("Content-Disposition", "attachment; filename=heritrix-test.bin"); diff --git a/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java b/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java new file mode 100644 index 00000000..4bffe9fa --- /dev/null +++ b/modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java @@ -0,0 +1,97 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.behaviors; + +import org.archive.modules.extractor.UriErrorLoggerModule; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Repeatedly clicks a "next page" button to step through paginated content, + * extracting links after each click. Use as a replacement for ExtractLinksBehavior. + */ +public class PaginationBehavior implements Behavior { + private final ExtractLinksBehavior extractLinks; + private final AtomicLong pagesClicked = new AtomicLong(); + private String selector = "button[title=\"Next\"]"; + private int maxPages = 100; + private long loadTimeout = 5000; + + public PaginationBehavior(UriErrorLoggerModule loggerModule) { + this.extractLinks = new ExtractLinksBehavior(loggerModule); + } + + /** + * CSS selector for the next-page button. + */ + public void setSelector(String selector) { + this.selector = selector; + } + + /** + * Maximum number of times to click the next-page button. + */ + public void setMaxPages(int maxPages) { + this.maxPages = maxPages; + } + + /** + * Maximum time to wait for the page to change after each click, in milliseconds. + */ + public void setLoadTimeout(long loadTimeout) { + this.loadTimeout = loadTimeout; + } + + @Override + public void run(Page page) { + extractLinks.run(page); // links on the initial page + + for (int i = 0; i < maxPages; i++) { + // click the next-page button, then wait for the DOM to change (or give up after loadTimeout) + Boolean clicked = page.evalPromise(/* language=JavaScript */ """ + (selector, timeout) => new Promise(resolve => { + const button = document.querySelector(selector); + if (!button || button.disabled) { + resolve(false); + return; + } + const observer = new MutationObserver(() => { + observer.disconnect(); + setTimeout(() => resolve(true), 100); + }); + observer.observe(document.body, {childList: true, subtree: true}); + setTimeout(() => { + observer.disconnect(); + resolve(true); + }, timeout); + button.click(); + })""", selector, loadTimeout); + if (!Boolean.TRUE.equals(clicked)) break; + pagesClicked.incrementAndGet(); + + extractLinks.run(page); + } + } + + @Override + public String report() { + return Behavior.super.report() + " Pages clicked: " + pagesClicked.get() + "\n"; + } +}