From 92ffdede0d017472e56dac64f622efdab5d3f3e2 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 15:03:52 +0900 Subject: [PATCH 1/3] Link to javadoc.io for more recent api docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1a47653..2dac370f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Heritrix [![Build Status](https://travis-ci.org/internetarchive/heritrix3.svg?branch=master)](https://travis-ci.org/internetarchive/heritrix3) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix) -[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](http://builds.archive.org/javadoc/heritrix-3.2.0/) +[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine) [![LICENSE](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](./LICENSE) ## Introduction @@ -21,7 +21,7 @@ load your crawl will place on seed sites and set politeness policies accordingly - [Developer Manual](http://crawler.archive.org/articles/developer_manual/index.html) - [REST API documentation](https://heritrix.readthedocs.io/en/latest/api.html) -- [JavaDoc](http://builds.archive.org/javadoc/heritrix-3.2.0/) (n.b. Javadoc currently out of date) +- JavaDoc: [engine](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine), [modules](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-modules), [commons](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-commons), [contrib](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-contrib) ## Latest Releases From 36003b8626273fda6a643d719f2d973ef0393655 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 17:36:16 +0900 Subject: [PATCH 2/3] Fix digest authentication In Restlet 2 it appears we need to use DigestAuthenticator. (Previously both digest and basic auth were handled by the same Guard class.) --- .../src/main/java/org/archive/crawler/Heritrix.java | 6 +++--- .../org/archive/crawler/restlet/RateLimitGuard.java | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index ee0fcf79..5714b9f6 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -351,9 +351,9 @@ public class Heritrix { MapVerifier verifier = new MapVerifier(); verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); - ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(), - ChallengeScheme.HTTP_DIGEST, "Authentication Required"); - guard.setVerifier(verifier); + RateLimitGuard guard = new RateLimitGuard(component.getContext().createChildContext(), + "Authentication Required", UUID.randomUUID().toString()); + guard.setWrappedVerifier(verifier); guard.setNext(new EngineApplication(engine)); component.getDefaultHost().attach(guard); diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index 042fd2ea..7d9cd703 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -18,13 +18,12 @@ */ package org.archive.crawler.restlet; -import java.util.logging.Logger; - import org.restlet.Context; import org.restlet.Request; import org.restlet.Response; -import org.restlet.data.ChallengeScheme; -import org.restlet.security.ChallengeAuthenticator; +import org.restlet.ext.crypto.DigestAuthenticator; + +import java.util.logging.Logger; /** * ChallengeAuthenticator that slows and logs failed authentication attempts, to make @@ -32,15 +31,15 @@ import org.restlet.security.ChallengeAuthenticator; * * @author gojomo */ -public class RateLimitGuard extends ChallengeAuthenticator { +public class RateLimitGuard extends DigestAuthenticator { private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000; private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); protected long lastFailureTime = 0; - public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) throws IllegalArgumentException { - super(context, scheme, realm); + public RateLimitGuard(Context context, String realm, String serverKey) throws IllegalArgumentException { + super(context, realm, serverKey); } @Override From 38c44e8e8e6189bdd46e7ffeb80fa7657eb27fd5 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 17:50:35 +0900 Subject: [PATCH 3/3] Add missing UUID import (interactive commit fail) --- engine/src/main/java/org/archive/crawler/Heritrix.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 5714b9f6..dd9e208e 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -33,12 +33,7 @@ import java.net.UnknownHostException; import java.security.KeyStore; import java.security.MessageDigest; import java.security.cert.Certificate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; import java.util.logging.LogManager; import java.util.logging.Logger;