Add PaginationBehavior: repeatedly clicks next-page and extracts links

commons/src/main/java/org/archive/net/webdriver/BiDiJson.java

@@ -185,10 +185,13 @@ class BiDiJson {
     }
 
     private static Class<?> getSubclassByTypeName(Class<?> type, String typeName) {
+        Class<?> fallback = null;
         for (Class<?> subclass : type.getPermittedSubclasses()) {
             TypeName annotation = subclass.getDeclaredAnnotation(TypeName.class);
             if (annotation != null && annotation.value().equals(typeName)) return subclass;
+            if (subclass.getDeclaredAnnotation(UnknownTypeFallback.class) != null) fallback = subclass;
         }
+        if (fallback != null) return fallback;
         throw new UnsupportedOperationException("Unsupported 'type' for " + type + ": " + typeName);
     }
 
@@ -208,6 +211,14 @@ class BiDiJson {
         String value();
     }
 
+    /**
+     * Marks the subclass to deserialize into when the 'type' field matches no other permitted subclass.
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target(ElementType.TYPE)
+    @interface UnknownTypeFallback {
+    }
+
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.RECORD_COMPONENT)
     @interface PropertyName {

commons/src/main/java/org/archive/net/webdriver/Script.java

@@ -58,6 +58,8 @@ public interface Script extends BiDiModule {
     sealed interface LocalValue {
         static LocalValue from(Object object) {
             if (object instanceof Number number) return new NumberValue(number);
+            if (object instanceof String string) return new StringValue(string);
+            if (object instanceof Boolean bool) return new BooleanValue(bool);
             throw new IllegalArgumentException("Unsupported local value type: " + object.getClass());
         }
     }
@@ -82,6 +84,34 @@ public interface Script extends BiDiModule {
         }
     }
 
+    @BiDiJson.TypeName("boolean")
+    record BooleanValue(boolean value) implements RemoteValue, LocalValue {
+        @Override
+        public Boolean javaValue() {
+            return value;
+        }
+    }
+
+    @BiDiJson.TypeName("null")
+    record NullValue() implements RemoteValue, LocalValue {
+        @Override
+        public Object javaValue() {
+            return null;
+        }
+    }
+
+    /**
+     * Catch-all for RemoteValue types that aren't modelled here, as page scripts can return any
+     * type the protocol defines. The value is the raw JSON 'value' field, if there was one.
+     */
+    @BiDiJson.UnknownTypeFallback
+    record UnknownValue(String type, Object value) implements RemoteValue {
+        @Override
+        public Object javaValue() {
+            return value;
+        }
+    }
+
     @BiDiJson.TypeName("array")
     record ArrayRemoteValue(List<RemoteValue> value) implements RemoteValue, LocalValue {
         @Override

engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java

@@ -3,6 +3,7 @@ package org.archive.crawler.processor;
 import com.sun.net.httpserver.HttpServer;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.modules.*;
+import org.archive.modules.behaviors.PaginationBehavior;
 import org.archive.modules.fetcher.DefaultServerCache;
 import org.archive.modules.fetcher.FetchHTTP2;
 import org.archive.net.UURIFactory;
@@ -62,6 +63,27 @@ class BrowserProcessorTest {
         assertEquals("/*hello world*/", gzip.getRecorder().getContentReplayPrefixString(100));
     }
 
+    @Test
+    public void testPaginationBehavior() throws IOException, InterruptedException {
+        var defaultBehaviors = browserProcessor.getBehaviors();
+        browserProcessor.setBehaviors(List.of(new PaginationBehavior(null)));
+        try {
+            CrawlURI crawlURI = newCrawlURI(baseUrl + "paginated");
+            fetcher.process(crawlURI);
+            assertEquals(200, crawlURI.getFetchStatus());
+            browserProcessor.innerProcess(crawlURI);
+
+            var paths = new HashSet<String>();
+            for (CrawlURI link : crawlURI.getOutLinks()) {
+                paths.add(link.getUURI().getPath());
+            }
+            assertEquals(Set.of("/item1", "/item2", "/item3"), paths,
+                    "should have extracted the links from every page of the pagination");
+        } finally {
+            browserProcessor.setBehaviors(defaultBehaviors);
+        }
+    }
+
     @Test
     public void testDownload() throws IOException, InterruptedException {
         CrawlURI crawlURI = newCrawlURI(baseUrl + "download.bin");
@@ -143,6 +165,18 @@ class BrowserProcessorTest {
                     body = "body { color: red; background: url(bg.jpg); }";
                     contentType = "text/css";
                 }
+                case "/paginated" -> body = """
+                        <div id=items><a href="/item1">item</a></div>
+                        <button id=next title="Next" onclick="nextPage()">Next</button>
+                        <script>
+                            let page = 1;
+                            function nextPage() {
+                                page++;
+                                document.getElementById('items').innerHTML =
+                                    '<a href="/item' + page + '">item</a>';
+                                if (page >= 3) document.getElementById('next').disabled = true;
+                            }
+                        </script>""";
                 case "/download.bin" -> {
                     body = "sample-download-file";
                     exchange.getResponseHeaders().add("Content-Disposition", "attachment; filename=heritrix-test.bin");

modules/src/main/java/org/archive/modules/behaviors/PaginationBehavior.java

@@ -0,0 +1,97 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.modules.behaviors;
+
+import org.archive.modules.extractor.UriErrorLoggerModule;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Repeatedly clicks a "next page" button to step through paginated content,
+ * extracting links after each click. Use as a replacement for ExtractLinksBehavior.
+ */
+public class PaginationBehavior implements Behavior {
+    private final ExtractLinksBehavior extractLinks;
+    private final AtomicLong pagesClicked = new AtomicLong();
+    private String selector = "button[title=\"Next\"]";
+    private int maxPages = 100;
+    private long loadTimeout = 5000;
+
+    public PaginationBehavior(UriErrorLoggerModule loggerModule) {
+        this.extractLinks = new ExtractLinksBehavior(loggerModule);
+    }
+
+    /**
+     * CSS selector for the next-page button.
+     */
+    public void setSelector(String selector) {
+        this.selector = selector;
+    }
+
+    /**
+     * Maximum number of times to click the next-page button.
+     */
+    public void setMaxPages(int maxPages) {
+        this.maxPages = maxPages;
+    }
+
+    /**
+     * Maximum time to wait for the page to change after each click, in milliseconds.
+     */
+    public void setLoadTimeout(long loadTimeout) {
+        this.loadTimeout = loadTimeout;
+    }
+
+    @Override
+    public void run(Page page) {
+        extractLinks.run(page); // links on the initial page
+
+        for (int i = 0; i < maxPages; i++) {
+            // click the next-page button, then wait for the DOM to change (or give up after loadTimeout)
+            Boolean clicked = page.evalPromise(/* language=JavaScript */ """
+                    (selector, timeout) => new Promise(resolve => {
+                        const button = document.querySelector(selector);
+                        if (!button || button.disabled) {
+                            resolve(false);
+                            return;
+                        }
+                        const observer = new MutationObserver(() => {
+                            observer.disconnect();
+                            setTimeout(() => resolve(true), 100);
+                        });
+                        observer.observe(document.body, {childList: true, subtree: true});
+                        setTimeout(() => {
+                            observer.disconnect();
+                            resolve(true);
+                        }, timeout);
+                        button.click();
+                    })""", selector, loadTimeout);
+            if (!Boolean.TRUE.equals(clicked)) break;
+            pagesClicked.incrementAndGet();
+
+            extractLinks.run(page);
+        }
+    }
+
+    @Override
+    public String report() {
+        return Behavior.super.report() + "    Pages clicked: " + pagesClicked.get() + "\n";
+    }
+}