Add Browser processor and behaviors to bean reference

docs/bean-reference.rst

@@ -498,6 +498,24 @@ TrapSuppressExtractor
 
 .. bean-doc:: ../modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java
 
+Browser and Behaviors
+---------------------
+
+Browser
+~~~~~~~~~~~~~~~~~
+
+.. bean-doc:: ../engine/src/main/java/org/archive/crawler/processor/Browser.java
+
+ExtractLinks
+~~~~~~~~~~~~
+
+.. bean-doc:: ../modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java
+
+ScrollDown
+~~~~~~~~~~
+
+.. bean-doc:: ../modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java
+
 Post-Processors
 ---------------
 

engine/src/main/java/org/archive/crawler/processor/Browser.java

@@ -68,9 +68,9 @@ import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUC
 import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS;
 
 /**
- * Processor which opens a web page in a real web browser via WebDriver BiDi and runs {@link Behavior} scripts.
- * Subresources loaded by the browser are recorded using a MITM proxy. Must be used in conjunction with
- * {@link FetchHTTP2}.
+ * Processor which opens a web page in a local web browser via WebDriver BiDi and runs {@link Behavior} scripts.
+ * Subresources loaded by the browser are recorded using a recording proxy. Must be used in conjunction with
+ * {@link FetchHTTP2}. Normally defined in the FetchChain after the link extractors.
  */
 public class Browser extends Processor {
     private static final System.Logger logger = System.getLogger(Browser.class.getName());
@@ -297,7 +297,7 @@ public class Browser extends Processor {
     }
 
     /**
-     * Sets a list of {@link Behavior}s to run on each page.
+     * A list of {@link Behavior}s to run on each page.
      */
     public void setBehaviors(List<Behavior> behaviors) {
         this.behaviors = behaviors;
@@ -308,9 +308,9 @@ public class Browser extends Processor {
     }
 
     /**
-     * Sets the webdriver executable to launch. If null, will try several common paths.
+     * Webdriver executable to launch. If null, will try several common paths.
      * <p>
-     * Firefox can be used directly as implements WebDriver BiDI natively. To use Chrome set this to a
+     * Firefox can be used directly as it implements WebDriver BiDI natively. To use Chrome set this to a
      * <a href="https://developer.chrome.com/docs/chromedriver">ChromeDriver</a> executable.
      */
     public void setExecutable(String executable) {
@@ -325,6 +325,9 @@ public class Browser extends Processor {
         return options;
     }
 
+    /**
+     * Extra command-line options to be passed to the webdriver executable.
+     */
     public void setOptions(List<String> options) {
         this.options = options;
     }
@@ -334,7 +337,7 @@ public class Browser extends Processor {
     }
 
     /**
-     * Sets the maximum number of web pages that can be open in the browser at once.
+     * Maximum number of web pages that can be open in the browser at once.
      */
     public void setConcurrency(int concurrency) {
         this.concurrency = concurrency;

modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java

@@ -30,6 +30,9 @@ import org.archive.net.UURIFactory;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 
+/**
+ * Extracts navigation links from the loaded page using JavaScript.
+ */
 public class ExtractLinks implements Behavior {
     private final AtomicLong numberOfLinksExtracted = new AtomicLong(0);
     private final UriErrorLoggerModule loggerModule;

modules/src/main/java/org/archive/modules/behaviors/ScrollDown.java

@@ -26,10 +26,16 @@ public class ScrollDown implements Behavior {
     private long timeout = 5000;
     private int scrollInterval = 50;
 
+    /**
+     * Maximum time to wait to reach the bottom of the page, in milliseconds.
+     */
     public void setTimeout(long timeout) {
         this.timeout = timeout;
     }
 
+    /**
+     * How many milliseconds to wait between each scroll step.
+     */
     public void setScrollInterval(int scrollInterval) {
         this.scrollInterval = scrollInterval;
     }