From cf447f3bd9579f099aee211ee9479ea650625637 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Mon, 9 Jun 2025 15:19:09 +0900
Subject: [PATCH] Add Browser processor and behaviors to bean reference

---
 docs/bean-reference.rst                        | 18 ++++++++++++++++++
 .../org/archive/crawler/processor/Browser.java | 17 ++++++++++-------
 .../modules/behaviors/ExtractLinks.java        |  3 +++
 .../archive/modules/behaviors/ScrollDown.java  |  6 ++++++
 4 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/docs/bean-reference.rst b/docs/bean-reference.rst
index 48af1a52..75132d9d 100644
--- a/docs/bean-reference.rst
+++ b/docs/bean-reference.rst
@@ -498,6 +498,24 @@ TrapSuppressExtractor
 
 .. bean-doc:: ../modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java
 
+Browser and Behaviors
+---------------------
+
+Browser
+~~~~~~~~~~~~~~~~~
+
+.. bean-doc:: ../engine/src/main/java/org/archive/crawler/processor/Browser.java
+
+ExtractLinks
+~~~~~~~~~~~~
+
+.. bean-doc:: ../modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java
+
+ScrollDown
+~~~~~~~~~~
+
+.. bean-doc:: ../modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java
+
 Post-Processors
 ---------------
 
diff --git a/engine/src/main/java/org/archive/crawler/processor/Browser.java b/engine/src/main/java/org/archive/crawler/processor/Browser.java
index da44fe46..f2221636 100644
--- a/engine/src/main/java/org/archive/crawler/processor/Browser.java
+++ b/engine/src/main/java/org/archive/crawler/processor/Browser.java
@@ -68,9 +68,9 @@ import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUC
 import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS;
 
 /**
- * Processor which opens a web page in a real web browser via WebDriver BiDi and runs {@link Behavior} scripts.
- * Subresources loaded by the browser are recorded using a MITM proxy. Must be used in conjunction with
- * {@link FetchHTTP2}.
+ * Processor which opens a web page in a local web browser via WebDriver BiDi and runs {@link Behavior} scripts.
+ * Subresources loaded by the browser are recorded using a recording proxy. Must be used in conjunction with
+ * {@link FetchHTTP2}. Normally defined in the FetchChain after the link extractors.
  */
 public class Browser extends Processor {
     private static final System.Logger logger = System.getLogger(Browser.class.getName());
@@ -297,7 +297,7 @@ public class Browser extends Processor {
     }
 
     /**
-     * Sets a list of {@link Behavior}s to run on each page.
+     * A list of {@link Behavior}s to run on each page.
      */
     public void setBehaviors(List<Behavior> behaviors) {
         this.behaviors = behaviors;
@@ -308,9 +308,9 @@ public class Browser extends Processor {
     }
 
     /**
-     * Sets the webdriver executable to launch. If null, will try several common paths.
+     * Webdriver executable to launch. If null, will try several common paths.
      * <p>
-     * Firefox can be used directly as implements WebDriver BiDI natively. To use Chrome set this to a
+     * Firefox can be used directly as it implements WebDriver BiDI natively. To use Chrome set this to a
      * <a href="https://developer.chrome.com/docs/chromedriver">ChromeDriver</a> executable.
      */
     public void setExecutable(String executable) {
@@ -325,6 +325,9 @@ public class Browser extends Processor {
         return options;
     }
 
+    /**
+     * Extra command-line options to be passed to the webdriver executable.
+     */
     public void setOptions(List<String> options) {
         this.options = options;
     }
@@ -334,7 +337,7 @@ public class Browser extends Processor {
     }
 
     /**
-     * Sets the maximum number of web pages that can be open in the browser at once.
+     * Maximum number of web pages that can be open in the browser at once.
      */
     public void setConcurrency(int concurrency) {
         this.concurrency = concurrency;
diff --git a/modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java b/modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java
index 22e67887..129d9651 100644
--- a/modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java
+++ b/modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java
@@ -30,6 +30,9 @@ import org.archive.net.UURIFactory;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 
+/**
+ * Extracts navigation links from the loaded page using JavaScript.
+ */
 public class ExtractLinks implements Behavior {
     private final AtomicLong numberOfLinksExtracted = new AtomicLong(0);
     private final UriErrorLoggerModule loggerModule;
diff --git a/modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java b/modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java
index d374373e..111b5bb0 100644
--- a/modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java
+++ b/modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java
@@ -26,10 +26,16 @@ public class ScrollDown implements Behavior {
     private long timeout = 5000;
     private int scrollInterval = 50;
 
+    /**
+     * Maximum time to wait to reach the bottom of the page, in milliseconds.
+     */
     public void setTimeout(long timeout) {
         this.timeout = timeout;
     }
 
+    /**
+     * How many milliseconds to wait between each scroll step.
+     */
     public void setScrollInterval(int scrollInterval) {
         this.scrollInterval = scrollInterval;
     }