lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-28 08:35:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into upgrade-bdb-je

Browse Source
This commit is contained in:
Andrew Jackson
2019-08-12 16:21:29 +01:00
parent 7dab6ec275 38c44e8e8e
commit f63570f977
3 changed files with 12 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -1,7 +1,7 @@
# Heritrix
[![Build Status](https://travis-ci.org/internetarchive/heritrix3.svg?branch=master)](https://travis-ci.org/internetarchive/heritrix3)
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix)
[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](http://builds.archive.org/javadoc/heritrix-3.2.0/)
[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine)
[![LICENSE](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](./LICENSE)
## Introduction
@@ -21,7 +21,7 @@ load your crawl will place on seed sites and set politeness policies accordingly
- [Developer Manual](http://crawler.archive.org/articles/developer_manual/index.html)
- [REST API documentation](https://heritrix.readthedocs.io/en/latest/api.html)
- [JavaDoc](http://builds.archive.org/javadoc/heritrix-3.2.0/) (n.b. Javadoc currently out of date)
- JavaDoc: [engine](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine), [modules](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-modules), [commons](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-commons), [contrib](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-contrib)
## Latest Releases
engine/src/main/java/org/archive/crawler/Heritrix.java
+4 -9
View File
@@ -33,12 +33,7 @@ import java.net.UnknownHostException;
import java.security.KeyStore;
import java.security.MessageDigest;
import java.security.cert.Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TimeZone;
import java.util.*;
import java.util.logging.LogManager;
import java.util.logging.Logger;
@@ -351,9 +346,9 @@ public class Heritrix {
MapVerifier verifier = new MapVerifier();
verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray());
ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(),
ChallengeScheme.HTTP_DIGEST, "Authentication Required");
guard.setVerifier(verifier);
RateLimitGuard guard = new RateLimitGuard(component.getContext().createChildContext(),
"Authentication Required", UUID.randomUUID().toString());
guard.setWrappedVerifier(verifier);
guard.setNext(new EngineApplication(engine));
component.getDefaultHost().attach(guard);
engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java
+6 -7
View File
@@ -18,13 +18,12 @@
*/
package org.archive.crawler.restlet;
import java.util.logging.Logger;
import org.restlet.Context;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.ChallengeScheme;
import org.restlet.security.ChallengeAuthenticator;
import org.restlet.ext.crypto.DigestAuthenticator;
import java.util.logging.Logger;
/**
* ChallengeAuthenticator that slows and logs failed authentication attempts, to make
@@ -32,15 +31,15 @@ import org.restlet.security.ChallengeAuthenticator;
*
* @author gojomo
*/
public class RateLimitGuard extends ChallengeAuthenticator {
public class RateLimitGuard extends DigestAuthenticator {
private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000;
private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName());
protected long lastFailureTime = 0;
public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) throws IllegalArgumentException {
super(context, scheme, realm);
public RateLimitGuard(Context context, String realm, String serverKey) throws IllegalArgumentException {
super(context, realm, serverKey);
}
@Override