From eaf72b5ce299c3b9dd4f03dcb00af0ddd366e644 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Sun, 25 May 2025 00:29:14 +0900 Subject: [PATCH] Add `--web-auth basic` command-line option This option enables HTTP Basic authentication for the web interface instead of the default Digest authentication. This is useful when running Heritrix behind a reverse proxy that adds external authentication as typically they don't support Digest auth for the upstream server. #641 --- CHANGELOG.md | 6 ++++ docs/operating.rst | 2 ++ .../java/org/archive/crawler/Heritrix.java | 32 +++++++++++++++++-- .../crawler/restlet/RateLimitGuard.java | 4 ++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a8d2cea..e0d6c72b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ [Full Changelog](https://github.com/internetarchive/heritrix3/compare/3.9.0...HEAD) +#### New features + +- **Basic web auth:** You can now switch the web interface from Digest authentication to Basic authentication + with the `--web-auth basic` command-line option. This is useful when running Heritrix behind a reverse proxy that + adds external authentication. + #### Fixes - **Code editor:** The configuration editor and script console were upgraded to CodeMirror 6. This resolves some browser diff --git a/docs/operating.rst b/docs/operating.rst index 92680fcf..8638ac4c 100644 --- a/docs/operating.rst +++ b/docs/operating.rst @@ -42,6 +42,8 @@ Command-line Options Specifies a keystore path, keystore password, and key password for HTTPS use. Separate the values with commas and do not include whitespace. By default Heritrix will generate a self-signed certificate the first time it is run. +--web-auth digest|basic + Authentication mode for the web interface. **Default:** ``digest`` Environment Variables ~~~~~~~~~~~~~~~~~~~~~ diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index fc830041..082036b6 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -56,7 +56,9 @@ import org.archive.util.KeyTool; import org.restlet.Component; import org.restlet.Context; import org.restlet.Server; +import org.restlet.data.ChallengeScheme; import org.restlet.data.Protocol; +import org.restlet.security.ChallengeAuthenticator; import org.restlet.security.MapVerifier; @@ -111,6 +113,13 @@ public class Heritrix { */ private static final String STARTLOG = "heritrix_dmesg.log"; + private enum AuthMode { + DIGEST, BASIC; + + public String toString() { + return name().toLowerCase(); + } + } private static void usage(PrintStream out, String[] args) { HelpFormatter hf = new HelpFormatter(); @@ -122,6 +131,8 @@ public class Heritrix { private static Options options() { Options options = new Options(); options.addOption("h", "help", true, "Usage information." ); + options.addOption(null, "web-auth", true, "Authentication mode for the" + + " web interface: " + Arrays.toString(AuthMode.values())); options.addOption("a", "web-admin", true, "REQUIRED. Specifies the " + "authorization username and password which must be supplied to " + "access the web interface. This may be of the form " + @@ -237,6 +248,7 @@ public class Heritrix { // DEFAULTS until changed by cmd-line options int port = 8443; Set bindHosts = new HashSet(); + AuthMode authMode = AuthMode.DIGEST; String authLogin = "admin"; String authPassword = null; String keystorePath; @@ -330,6 +342,16 @@ public class Heritrix { System.setProperty("https.proxyPort", proxyPort); } + if (cl.hasOption("web-auth")) { + try { + authMode = AuthMode.valueOf(cl.getOptionValue("web-auth").toUpperCase()); + } catch (IllegalArgumentException e) { + System.err.println("Unsupported --web-auth value '" + cl.getOptionValue("web-auth") + + "' (must be one of " + Arrays.toString(AuthMode.values()) + ")"); + System.exit(1); + } + } + // Restlet will reconfigure logging according to the system property // so we must set it for -l to work properly System.setProperty("java.util.logging.config.file", properties.getPath()); @@ -373,9 +395,13 @@ public class Heritrix { MapVerifier verifier = new MapVerifier(); verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); - RateLimitGuard guard = new RateLimitGuard(component.getContext().createChildContext(), - "Authentication Required", UUID.randomUUID().toString()); - guard.setWrappedVerifier(verifier); + Context guardContext = component.getContext().createChildContext(); + ChallengeAuthenticator guard = switch (authMode) { + case BASIC -> new ChallengeAuthenticator(guardContext, false, + ChallengeScheme.HTTP_BASIC, "Authentication Required", verifier); + case DIGEST -> new RateLimitGuard(guardContext, "Authentication Required", + UUID.randomUUID().toString(), verifier); + }; guard.setNext(new EngineApplication(engine)); component.getDefaultHost().attach(guard); diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index 937635e0..4bdbdf83 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -22,6 +22,7 @@ import org.restlet.Context; import org.restlet.Request; import org.restlet.Response; import org.restlet.ext.crypto.DigestAuthenticator; +import org.restlet.security.LocalVerifier; import java.util.logging.Logger; @@ -38,8 +39,9 @@ public class RateLimitGuard extends DigestAuthenticator { protected long lastFailureTime = 0; - public RateLimitGuard(Context context, String realm, String serverKey) throws IllegalArgumentException { + public RateLimitGuard(Context context, String realm, String serverKey, LocalVerifier verifier) throws IllegalArgumentException { super(context, realm, serverKey); + setWrappedVerifier(verifier); } @Override