lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 22:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #276 from internetarchive/jdk11-upgrade-jetty

Browse Source 
JDK11 support: upgrade to Jetty 9.4.19, Restlet 2.4.0 and drop JDK 7 support
This commit is contained in:
Noah Levitt
2019-08-07 10:48:51 -07:00
committed by GitHub
parent a1f2c9513a 97b9c3c91c
commit cefad96036
39 changed files with 427 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -5
View File
@@ -6,9 +6,6 @@ matrix:
include:
- jdk: oraclejdk8
dist: trusty
- jdk: openjdk7
dist: trusty
env: PROJECTS='--projects commons,modules,engine'
- jdk: openjdk8
- jdk: openjdk11
allow_failures:
@@ -22,14 +19,14 @@ before_install:
- "export _JAVA_OPTIONS=-Xmx1500m"
- "echo _JAVA_OPTIONS=$_JAVA_OPTIONS"
install: mvn dependency:resolve -B -V $PROJECTS
install: mvn dependency:resolve -B -V
cache:
directories:
- $HOME/.m2
script:
- travis_wait 30 mvn install $PROJECTS
- travis_wait 30 mvn install
after_failure:
- cat */target/surefire-reports/*.txt
commons/src/test/resources/log4j.xml
+1 -1
View File
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>
<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>
contrib/src/main/resources/log4j.xml
+1 -1
View File
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>
<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>
contrib/src/test/resources/log4j.xml
+1 -1
View File
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>
<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>
dist/src/main/conf/logging.properties
Vendored
+1 -1
View File
@@ -5,7 +5,7 @@
# ...and even less from the too-chatty-with-WARNINGs HttpClient library...
org.apache.commons.httpclient.level = SEVERE
org.restlet.Component.LogFilter.level = SEVERE
org.mortbay.log.level = SEVERE
org.eclipse.jetty.log.level = SEVERE
# ...but INFO for our classes, which reserve FINE/FINER/FINEST for bulk/trivia...
org.archive.level = INFO
dist/src/test/resources/log4j.xml
Vendored
+1 -1
View File
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>
<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>
engine/pom.xml
+34 -28
View File
@@ -25,44 +25,50 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<!-- we force the version of jetty rather than allowing restlet
to include it transitively as we need at least 9.4.12 for jdk11 -->
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.4.19.v20190610</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
<version>9.4.19.v20190610</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>9.4.19.v20190610</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet.ext.jetty</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-ajp</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet.ext.xml</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.restlet</groupId>
<artifactId>org.restlet</artifactId>
<version>1.1.10</version>
</dependency>
<dependency>
<groupId>com.noelios.restlet</groupId>
<artifactId>com.noelios.restlet</artifactId>
<version>1.1.10</version>
</dependency>
<dependency>
<groupId>com.noelios.restlet</groupId>
<artifactId>com.noelios.restlet.ext.jetty</artifactId>
<version>1.1.10</version>
<!-- for HTTP Digest auth -->
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet.ext.crypto</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
engine/src/main/java/org/archive/crawler/Heritrix.java
+20 -13
View File
@@ -57,10 +57,11 @@ import org.archive.crawler.restlet.RateLimitGuard;
import org.archive.util.ArchiveUtils;
import org.archive.util.KeyTool;
import org.restlet.Component;
import org.restlet.Guard;
import org.restlet.Server;
import org.restlet.data.ChallengeScheme;
import org.restlet.data.Protocol;
import org.restlet.security.ChallengeAuthenticator;
import org.restlet.security.MapVerifier;
/**
@@ -212,7 +213,7 @@ public class Heritrix {
"mailto, clsid, res, file, rtsp, about");
}
String maxFormSize = "org.mortbay.jetty.Request.maxFormContentSize";
String maxFormSize = "org.eclipse.jetty.server.Request.maxFormContentSize";
if (System.getProperty(maxFormSize) == null) {
System.setProperty(maxFormSize, "52428800");
}
@@ -334,24 +335,30 @@ public class Heritrix {
try {
engine = new Engine(jobsDir);
component = new Component();
if(bindHosts.isEmpty()) {
// listen all addresses
setupServer(port, null, keystorePath, keystorePassword, keyPassword);
setupServer(component, port, null, keystorePath, keystorePassword, keyPassword);
} else {
// bind only to declared addresses, or just 'localhost'
for(String address : bindHosts) {
setupServer(port, address, keystorePath, keystorePassword, keyPassword);
setupServer(component, port, address, keystorePath, keystorePassword, keyPassword);
}
}
component.getClients().add(Protocol.FILE);
component.getClients().add(Protocol.CLAP);
Guard guard = new RateLimitGuard(null,
component.getClients().add(Protocol.CLAP);
MapVerifier verifier = new MapVerifier();
verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray());
ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(),
ChallengeScheme.HTTP_DIGEST, "Authentication Required");
guard.getSecrets().put(authLogin, authPassword.toCharArray());
component.getDefaultHost().attach(guard);
guard.setVerifier(verifier);
guard.setNext(new EngineApplication(engine));
component.getDefaultHost().attach(guard);
component.start();
startupOut.println("engine listening at port "+port);
startupOut.println("operator login set per " +
((aOption.startsWith("@")) ? "file "+aOption : "command-line"));
@@ -457,16 +464,16 @@ public class Heritrix {
/**
* Create an HTTPS restlet Server instance matching the given parameters.
*
*
* @param component
* @param port
* @param address
* @param keystorePath
* @param keystorePassword
* @param keyPassword
*/
protected void setupServer(int port, String address, String keystorePath, String keystorePassword, String keyPassword) {
Server server = new Server(Protocol.HTTPS,address,port,null);
component.getServers().add(server);
protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) {
Server server = component.getServers().add(Protocol.HTTPS, address, port);
server.getContext().getParameters().add("keystorePath", keystorePath);
server.getContext().getParameters().add("keystorePassword", keystorePassword);
server.getContext().getParameters().add("keyPassword", keyPassword);
engine/src/main/java/org/archive/crawler/restlet/BaseResource.java
+2 -43
View File
@@ -19,55 +19,14 @@
package org.archive.crawler.restlet;
import java.util.List;
import org.restlet.Context;
import org.restlet.data.MediaType;
import org.restlet.data.Preference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Resource;
import org.restlet.resource.Variant;
import org.restlet.resource.ServerResource;
/**
* Abstract {@code Resource} with common shared functionality.
*
* @author nlevitt
*/
public abstract class BaseResource extends Resource {
public BaseResource(Context ctx, Request req, Response res) {
super(ctx, req, res);
}
/**
* If client can accept text/html, always prefer it. WebKit-based browsers
* claim to want application/xml, but we don't want to give it to them. See
* <a href="https://webarchive.jira.com/browse/HER-1603">https://webarchive.jira.com/browse/HER-1603</a>
*/
public Variant getPreferredVariant() {
boolean addExplicitTextHtmlPreference = false;
for (Preference<MediaType> mediaTypePreference: getRequest().getClientInfo().getAcceptedMediaTypes()) {
if (mediaTypePreference.getMetadata().equals(MediaType.TEXT_HTML)) {
mediaTypePreference.setQuality(Float.MAX_VALUE);
addExplicitTextHtmlPreference = false;
break;
} else if (mediaTypePreference.getMetadata().includes(MediaType.TEXT_HTML)) {
addExplicitTextHtmlPreference = true;
}
}
if (addExplicitTextHtmlPreference) {
List<Preference<MediaType>> acceptedMediaTypes = getRequest().getClientInfo().getAcceptedMediaTypes();
acceptedMediaTypes.add(new Preference<MediaType>(MediaType.TEXT_HTML, Float.MAX_VALUE));
getRequest().getClientInfo().setAcceptedMediaTypes(acceptedMediaTypes);
}
return super.getPreferredVariant();
}
public abstract class BaseResource extends ServerResource {
protected String getStaticRef(String resource) {
String rootRef = getRequest().getRootRef().toString();
return rootRef + "/engine/static/" + resource;
engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java
+18 -19
View File
@@ -19,16 +19,12 @@
package org.archive.crawler.restlet;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Set;
@@ -36,18 +32,18 @@ import org.apache.commons.lang.StringUtils;
import org.archive.crawler.restlet.models.BeansModel;
import org.archive.crawler.restlet.models.ViewModel;
import org.archive.spring.PathSharingContext;
import org.archive.util.TextUtils;
import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.representation.EmptyRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.WriterRepresentation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.restlet.representation.Variant;
import org.springframework.beans.BeanWrapperImpl;
import org.springframework.beans.BeansException;
@@ -69,12 +65,12 @@ public class BeanBrowseResource extends JobRelatedResource {
protected PathSharingContext appCtx;
protected String beanPath;
private Configuration _templateConfiguration;
public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);
@Override
public void init(Context ctx, Request req, Response res) throws ResourceException {
super.init(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
setModifiable(true); // accept POSTs
appCtx = cj.getJobContext();
beanPath = (String)req.getAttributes().get("beanPath");
if (beanPath!=null) {
@@ -99,13 +95,14 @@ public class BeanBrowseResource extends JobRelatedResource {
return _templateConfiguration;
}
public void acceptRepresentation(Representation entity) throws ResourceException {
@Override
protected Representation post(Representation entity, Variant variant) throws ResourceException {
if (appCtx == null) {
throw new ResourceException(404);
}
// copy op?
Form form = getRequest().getEntityAsForm();
Form form = new Form(entity);
beanPath = form.getFirstValue("beanPath");
String newVal = form.getFirstValue("newVal");
@@ -122,6 +119,7 @@ public class BeanBrowseResource extends JobRelatedResource {
ref.setPath(getBeansRefPath());
ref.addSegment(beanPath);
getResponse().redirectSeeOther(ref);
return new EmptyRepresentation();
}
public String getBeansRefPath() {
@@ -137,7 +135,8 @@ public class BeanBrowseResource extends JobRelatedResource {
return path;
}
public Representation represent(Variant variant) throws ResourceException {
@Override
public Representation get(Variant variant) throws ResourceException {
if (appCtx == null) {
throw new ResourceException(404);
}
@@ -209,8 +208,8 @@ public class BeanBrowseResource extends JobRelatedResource {
for(String name: appCtx.getBeanDefinitionNames()) {
addPresentableNestedNames(nestedNames, appCtx.getBean(name), alreadyWritten);
}
return new BeansModel(cj.getShortName(),
return new BeansModel(cj.getShortName(),
new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef().toString(),
beanPath,
bean,
engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java
+4 -4
View File
@@ -30,8 +30,8 @@ import org.apache.commons.lang.StringEscapeUtils;
import org.restlet.data.CharacterSet;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.resource.CharacterRepresentation;
import org.restlet.resource.FileRepresentation;
import org.restlet.representation.CharacterRepresentation;
import org.restlet.representation.FileRepresentation;
/**
* Representation wrapping a FileRepresentation, displaying its contents
@@ -40,9 +40,9 @@ import org.restlet.resource.FileRepresentation;
* @author gojomo
*/
public class EditRepresentation extends CharacterRepresentation {
protected FileRepresentation fileRepresentation;
protected FileRepresentation fileRepresentation;
protected EnhDirectoryResource dirResource;
public EditRepresentation(FileRepresentation representation, EnhDirectoryResource resource) {
super(MediaType.TEXT_HTML);
fileRepresentation = representation;
engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java
+10 -9
View File
@@ -26,19 +26,19 @@ import java.io.StringWriter;
import org.archive.crawler.framework.Engine;
import org.archive.util.TextUtils;
import org.restlet.Application;
import org.restlet.Directory;
import org.restlet.Redirector;
import org.restlet.Restlet;
import org.restlet.Router;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.Status;
import org.restlet.resource.Representation;
import org.restlet.resource.StringRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.StringRepresentation;
import org.restlet.resource.Directory;
import org.restlet.routing.Redirector;
import org.restlet.routing.Router;
import org.restlet.routing.Template;
import org.restlet.service.StatusService;
import org.restlet.util.Template;
/**
* Restlet Application for a Heritrix crawl 'Engine', which is aware of
@@ -56,7 +56,8 @@ public class EngineApplication extends Application {
setStatusService(new EngineStatusService());
}
public synchronized Restlet createRoot() {
@Override
public Restlet createInboundRoot() {
Router router = new Router(getContext());
router.attach("/",new Redirector(null,"/engine",Redirector.MODE_CLIENT_TEMPORARY));
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+22 -15
View File
@@ -30,21 +30,24 @@ import org.archive.crawler.framework.Engine;
import org.archive.crawler.restlet.models.EngineModel;
import org.archive.crawler.restlet.models.ViewModel;
import org.restlet.Context;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.representation.EmptyRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.WriterRepresentation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.restlet.representation.Variant;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import static org.restlet.data.MediaType.APPLICATION_XML;
/**
* Restlet Resource representing an Engine that may be used
* to assemble, launch, monitor, and manage crawls.
@@ -56,11 +59,12 @@ import freemarker.template.TemplateException;
public class EngineResource extends BaseResource {
private Configuration _templateConfiguration;
public EngineResource(Context ctx, Request req, Response res) {
super(ctx, req, res);
setModifiable(true);
@Override
public void init(Context ctx, Request req, Response res) {
super.init(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
getVariants().add(new Variant(APPLICATION_XML));
Configuration tmpltCfg = new Configuration();
tmpltCfg.setClassForTemplateLoading(this.getClass(),"");
@@ -74,10 +78,12 @@ public class EngineResource extends BaseResource {
public Configuration getTemplateConfiguration(){
return _templateConfiguration;
}
public Representation represent(Variant variant) throws ResourceException {
@Override
protected Representation get(Variant variant) throws ResourceException {
Representation representation;
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
if (variant.getMediaType() == APPLICATION_XML) {
representation = new WriterRepresentation(APPLICATION_XML) {
public void write(Writer writer) throws IOException {
XmlMarshaller.marshalDocument(writer, "engine", makeDataModel());
}
@@ -93,10 +99,10 @@ public class EngineResource extends BaseResource {
representation.setCharacterSet(CharacterSet.UTF_8);
return representation;
}
@Override
public void acceptRepresentation(Representation entity) throws ResourceException {
Form form = getRequest().getEntityAsForm();
protected Representation post(Representation entity, Variant variant) throws ResourceException {
Form form = new Form(entity);
String action = form.getFirstValue("action");
if("rescan".equals(action)) {
getEngine().findJobConfigs();
@@ -182,6 +188,7 @@ public class EngineResource extends BaseResource {
}
// default: redirect to GET self
getResponse().redirectSeeOther(getRequest().getOriginalRef());
return new EmptyRepresentation();
}
engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java
+23 -15
View File
@@ -26,10 +26,12 @@ import java.util.logging.Level;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.restlet.Context;
import org.restlet.Handler;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.Status;
import org.restlet.resource.Directory;
import org.restlet.resource.ServerResource;
/**
* Enhanced version of Restlet Directory, which allows the local
@@ -39,11 +41,11 @@ import org.restlet.data.Response;
*
* @author gojomo
*/
public abstract class EnhDirectory extends org.restlet.Directory {
public abstract class EnhDirectory extends Directory {
protected IOFileFilter editFilter = FileFilterUtils.falseFileFilter();
protected IOFileFilter pageFilter = FileFilterUtils.falseFileFilter();
protected IOFileFilter tailFilter = FileFilterUtils.falseFileFilter();
public EnhDirectory(Context context, Reference rootLocalReference) {
super(context, rootLocalReference);
// TODO Auto-generated constructor stub
@@ -55,21 +57,27 @@ public abstract class EnhDirectory extends org.restlet.Directory {
}
@Override
public Handler findTarget(Request request, Response response) {
Handler retVal;
synchronized(this) {
public void handle(Request request, Response response) {
synchronized (this) {
Reference oldRef = getRootRef();
setRootRef(determineRootRef(request));
try {
retVal = new EnhDirectoryResource(this, request, response);
} catch (IOException ioe) {
getLogger().log(Level.WARNING,
"Unable to find the directory's resource", ioe);
retVal = null;
super.handle(request, response);
} finally {
setRootRef(oldRef);
}
// XXX: FileRepresentation.isAvailable() returns false for empty files generating status 204 No Content
// which confuses browsers. Force it back it 200 OK.
if (response.getStatus() == Status.SUCCESS_NO_CONTENT) {
response.setStatus(Status.SUCCESS_OK);
}
setRootRef(oldRef);
}
return retVal;
}
@Override
public ServerResource create(Request request, Response response) {
return new EnhDirectoryResource();
}
protected abstract Reference determineRootRef(Request request);
engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java
+19 -28
View File
@@ -23,22 +23,18 @@ package org.archive.crawler.restlet;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import org.apache.commons.io.FileUtils;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.data.Status;
import org.restlet.resource.FileRepresentation;
import org.restlet.resource.Representation;
import org.restlet.data.*;
import org.restlet.engine.local.DirectoryServerResource;
import org.restlet.representation.EmptyRepresentation;
import org.restlet.representation.FileRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.Variant;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import com.noelios.restlet.local.DirectoryResource;
/**
* Enhanced version of Restlet DirectoryResource, adding ability to
@@ -46,20 +42,15 @@ import com.noelios.restlet.local.DirectoryResource;
*
* @author gojomo
*/
public class EnhDirectoryResource extends DirectoryResource {
public EnhDirectoryResource(EnhDirectory directory, Request request, Response response) throws IOException {
super(directory, request, response);
}
/**
public class EnhDirectoryResource extends DirectoryServerResource {
/**
* Add EditRepresentation as a variant when appropriate.
*
* @see com.noelios.restlet.local.DirectoryResource#getVariants()
* @see org.restlet.engine.local.DirectoryServerResource#getVariants()
*/
@Override
public List<Variant> getVariants() {
List<Variant> variants = super.getVariants();
List<Variant> variants = new LinkedList<>(super.getVariants(Method.GET));
Form f = getRequest().getResourceRef().getQueryAsForm();
String format = f.getFirstValue("format");
if("textedit".equals(format)) {
@@ -73,7 +64,7 @@ public class EnhDirectoryResource extends DirectoryResource {
} catch (Exception e) {
throw new RuntimeException(e);
}
variants = super.getVariants();
variants = new LinkedList<>(super.getVariants(Method.GET));
}
// wrap FileRepresentations in EditRepresentations
ListIterator<Variant> iter = variants.listIterator();
@@ -119,18 +110,18 @@ public class EnhDirectoryResource extends DirectoryResource {
}
protected EnhDirectory getEnhDirectory() {
return (EnhDirectory)getDirectory();
return (EnhDirectory) getDirectory();
}
/**
/**
* Accept a POST used to edit or create a file.
*
* @see org.restlet.resource.Resource#acceptRepresentation(org.restlet.resource.Representation)
* @see org.restlet.resource.ServerResource#post(Representation, Variant)
*/
public void acceptRepresentation(Representation entity)
throws ResourceException {
@Override
protected Representation post(Representation entity, Variant variant) throws ResourceException {
// TODO: only allowPost on valid targets
Form form = getRequest().getEntityAsForm();
Form form = new Form(entity);
String newContents = form.getFirstValue("contents");
EditRepresentation er;
try {
@@ -152,6 +143,6 @@ public class EnhDirectoryResource extends DirectoryResource {
Reference ref = getRequest().getOriginalRef().clone();
/// ref.setQuery(null);
getResponse().redirectSeeOther(ref);
return new EmptyRepresentation();
}
}
engine/src/main/java/org/archive/crawler/restlet/Flash.java
+2 -2
View File
@@ -29,10 +29,10 @@ import java.util.Map.Entry;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.Cookie;
import org.restlet.data.CookieSetting;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.util.Series;
/**
engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java
+12 -11
View File
@@ -40,8 +40,8 @@ import org.archive.crawler.framework.Engine;
import org.archive.util.TextUtils;
import org.restlet.Context;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.resource.ResourceException;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.BeanWrapperImpl;
@@ -58,18 +58,19 @@ public abstract class JobRelatedResource extends BaseResource {
private final static Logger LOGGER =
Logger.getLogger(JobRelatedResource.class.getName());
protected CrawlJob cj;
protected CrawlJob cj;
protected IdentityHashMap<Object, String> beanToNameMap;
public JobRelatedResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);
@Override
public void init(Context ctx, Request req, Response res) {
super.init(ctx, req, res);
cj = getEngine().getJob((String)req.getAttributes().get("job"));
if(cj==null) {
if(cj == null) {
throw new ResourceException(404);
}
}
protected Engine getEngine() {
return ((EngineApplication)getApplication()).getEngine();
}
@@ -95,13 +96,13 @@ public abstract class JobRelatedResource extends BaseResource {
return;
}
Reference baseRef = getRequest().getResourceRef().getBaseRef();
Reference baseRef = getRequest().getResourceRef().getBaseRef();
if (baseRef.getPath().endsWith("beans")) {
baseRef.setPath(baseRef.getPath() + "/");
}
if (getBeanToNameMap().containsKey(obj)) {
// this object is itself a named bean
Map<String, Object> bean = new LinkedHashMap<String, Object>();
engine/src/main/java/org/archive/crawler/restlet/JobResource.java
+16 -13
View File
@@ -38,12 +38,13 @@ import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.representation.EmptyRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.WriterRepresentation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.restlet.representation.Variant;
import freemarker.template.Configuration;
import freemarker.template.ObjectWrapper;
@@ -68,10 +69,10 @@ public class JobResource extends BaseResource {
protected CrawlJob cj;
public JobResource(Context ctx, Request req, Response res)
@Override
public void init(Context ctx, Request req, Response res)
throws ResourceException {
super(ctx, req, res);
setModifiable(true);
super.init(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
cj = getEngine().getJob(
@@ -88,7 +89,9 @@ public class JobResource extends BaseResource {
public Configuration getTemplateConfiguration(){
return _templateConfiguration;
}
public Representation represent(Variant variant) throws ResourceException {
@Override
public Representation get(Variant variant) throws ResourceException {
if (cj == null) {
throw new ResourceException(404);
}
@@ -184,19 +187,18 @@ public class JobResource extends BaseResource {
}
@Override
public void acceptRepresentation(Representation entity)
public Representation post(Representation entity, Variant variant)
throws ResourceException {
if (cj == null) {
throw new ResourceException(404);
}
// copy op?
Form form = null;
form = getRequest().getEntityAsForm();
Form form = new Form(entity);
String copyTo = form.getFirstValue("copyTo");
if (copyTo != null) {
copyJob(copyTo, "on".equals(form.getFirstValue("asProfile")));
return;
return new EmptyRepresentation();
}
AlertHandler.ensureStaticInitialization();
AlertThreadGroup.setThreadLogger(cj.getJobLogger());
@@ -241,6 +243,7 @@ public class JobResource extends BaseResource {
// default: redirect to GET self
getResponse().redirectSeeOther(getRequest().getOriginalRef());
return new EmptyRepresentation();
}
protected void copyJob(String copyTo, boolean asProfile)
engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java
+4 -4
View File
@@ -38,8 +38,8 @@ import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.resource.CharacterRepresentation;
import org.restlet.resource.FileRepresentation;
import org.restlet.representation.CharacterRepresentation;
import org.restlet.representation.FileRepresentation;
/**
* Representation wrapping a FileRepresentation, displaying its contents
@@ -50,7 +50,7 @@ import org.restlet.resource.FileRepresentation;
public class PagedRepresentation extends CharacterRepresentation {
// passed-in at construction
/** wrapped FileRepresentation **/
protected FileRepresentation fileRepresentation;
protected FileRepresentation fileRepresentation;
/** wrapped EnhDirectoryResource; used to formulate self-links **/
protected EnhDirectoryResource dirResource;
@@ -124,7 +124,7 @@ public class PagedRepresentation extends CharacterRepresentation {
/**
* Write the paged HTML.
*
* @see org.restlet.resource.Representation#write(java.io.Writer)
* @see org.restlet.representation.Representation#write(java.io.Writer)
*/
@Override
public void write(Writer writer) throws IOException {
engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java
+10 -14
View File
@@ -18,21 +18,21 @@
*/
package org.archive.crawler.restlet;
import java.util.Collection;
import java.util.logging.Logger;
import org.restlet.Context;
import org.restlet.Guard;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.ChallengeScheme;
import org.restlet.data.Request;
import org.restlet.security.ChallengeAuthenticator;
/**
* Guard that slows and logs failed authentication attempts, to make
* ChallengeAuthenticator that slows and logs failed authentication attempts, to make
* brute-force guessing attacks less feasible.
*
* @author gojomo
*/
public class RateLimitGuard extends Guard {
public class RateLimitGuard extends ChallengeAuthenticator {
private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000;
private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName());
@@ -43,14 +43,10 @@ public class RateLimitGuard extends Guard {
super(context, scheme, realm);
}
public RateLimitGuard(Context context, String realm, Collection<String> baseUris, String serverKey) {
super(context, realm, baseUris, serverKey);
}
@Override
public synchronized int authenticate(Request request) {
int retVal = super.authenticate(request);
if(retVal == AUTHENTICATION_INVALID) {
protected boolean authenticate(Request request, Response response) {
boolean succeeded = super.authenticate(request, response);
if (!succeeded) {
logger.warning("authentication failure "+request);
// wait until at least LAG has passed from last failure
// holding object lock the whole time, so no other checks
@@ -64,8 +60,8 @@ public class RateLimitGuard extends Guard {
// ignore
}
}
lastFailureTime = now + sleepMs;
lastFailureTime = now + sleepMs;
}
return retVal;
return succeeded;
}
}
engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java
+13 -11
View File
@@ -23,12 +23,12 @@ import java.io.File;
import org.restlet.Context;
import org.restlet.data.MediaType;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.representation.Representation;
import org.restlet.representation.StringRepresentation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.StringRepresentation;
import org.restlet.resource.Variant;
import org.restlet.representation.Variant;
/**
* Restlet Resource which generates fresh reports and then redirects
@@ -38,24 +38,26 @@ import org.restlet.resource.Variant;
*/
public class ReportGenResource extends JobRelatedResource {
protected String reportClass;
public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);
@Override
public void init(Context ctx, Request req, Response res) throws ResourceException {
super.init(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_PLAIN));
reportClass = (String)req.getAttributes().get("reportClass");
}
public Representation represent(Variant variant) throws ResourceException {
@Override
protected Representation get(Variant variant) throws ResourceException {
// generate report
if (cj == null || cj.getCrawlController() == null) {
throw new ResourceException(500);
}
File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass);
File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass);
if (f==null) {
throw new ResourceException(500);
}
// redirect
String relative = JobResource.getHrefPath(f,cj);
String relative = JobResource.getHrefPath(f, cj);
if(relative!=null) {
getResponse().redirectSeeOther("../"+relative+"?m="+f.lastModified());
return new StringRepresentation("");
engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java
+16 -15
View File
@@ -41,12 +41,12 @@ import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.representation.Representation;
import org.restlet.representation.WriterRepresentation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.restlet.representation.Variant;
import freemarker.template.Configuration;
import freemarker.template.ObjectWrapper;
@@ -81,10 +81,10 @@ public class ScriptResource extends JobRelatedResource {
protected String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0);
private Configuration _templateConfiguration;
public ScriptResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);
setModifiable(true);
@Override
public void init(Context ctx, Request req, Response res) throws ResourceException {
super.init(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
@@ -92,7 +92,7 @@ public class ScriptResource extends JobRelatedResource {
tmpltCfg.setClassForTemplateLoading(this.getClass(),"");
tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER);
setTemplateConfiguration(tmpltCfg);
scriptingConsole = new ScriptingConsole(cj);
}
public void setTemplateConfiguration(Configuration tmpltCfg) {
@@ -105,8 +105,8 @@ public class ScriptResource extends JobRelatedResource {
private ScriptingConsole scriptingConsole;
@Override
public void acceptRepresentation(Representation entity) throws ResourceException {
Form form = getRequest().getEntityAsForm();
public Representation post(Representation entity, Variant variant) throws ResourceException {
Form form = new Form(entity);
chosenEngine = form.getFirstValue("engine");
String script = form.getFirstValue("script");
if(StringUtils.isBlank(script)) {
@@ -121,10 +121,11 @@ public class ScriptResource extends JobRelatedResource {
//TODO: log script, results somewhere; job log INFO?
getResponse().setEntity(represent());
return get(variant);
}
public Representation represent(Variant variant) throws ResourceException {
@Override
public Representation get(Variant variant) throws ResourceException {
Representation representation;
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java
+1 -1
View File
@@ -36,7 +36,7 @@ import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import org.apache.commons.lang.StringUtils;
import org.restlet.util.XmlWriter;
import org.restlet.ext.xml.XmlWriter;
import org.xml.sax.SAXException;
/**
engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java
+5 -5
View File
@@ -22,10 +22,10 @@ package org.archive.crawler.selftest;
import java.io.IOException;
import org.archive.crawler.framework.CrawlJob;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.ServletHolder;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
/**
@@ -85,7 +85,7 @@ public class CheckpointSelfTest extends SelfTestBase {
private Server makeHttpServer(int port) throws Exception {
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost(HOST);
sc.setPort(port);
server.addConnector(sc);
engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java
+12 -11
View File
@@ -19,19 +19,20 @@
package org.archive.crawler.selftest;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.ServletHolder;
/**
* Test form-based authentication
@@ -60,17 +61,17 @@ public class FormAuthSelfTest
protected void startHttpServer() throws Exception {
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java
+12 -12
View File
@@ -19,20 +19,20 @@
package org.archive.crawler.selftest;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.ServletHolder;
/**
* Test form-based authentication
*
@@ -60,17 +60,17 @@ public class FormLoginSelfTest
protected void startHttpServer() throws Exception {
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java
+20 -21
View File
@@ -24,17 +24,16 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.security.Constraint;
import org.mortbay.jetty.security.ConstraintMapping;
import org.mortbay.jetty.security.HashUserRealm;
import org.mortbay.jetty.security.SecurityHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.eclipse.jetty.security.*;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.security.Password;
/**
* Test HTTP basic authentication
@@ -73,27 +72,27 @@ public class HttpAuthSelfTest
ConstraintMapping cm = new ConstraintMapping();
cm.setConstraint(constraint);
cm.setPathSpec("/basic/*");
UserStore userStore = new UserStore();
userStore.addUser("Mr. Happy Pants", new Password("xyzzy"), new String[]{"rule"});
HashLoginService loginService = new HashLoginService("Hyrule");
loginService.setUserStore(userStore);
HashUserRealm realm = new HashUserRealm();
realm.setName("Hyrule");
realm.put("Mr. Happy Pants", "xyzzy");
realm.addUserToRole("Mr. Happy Pants", "user");
SecurityHandler securityHandler = new SecurityHandler();
securityHandler.setUserRealm(realm);
ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler();
securityHandler.setLoginService(loginService);
securityHandler.setConstraintMappings(new ConstraintMapping[]{cm});
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
handlers.setHandlers(new Handler[] {
securityHandler,
rhandler,
servletHandler,
engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java
+7 -7
View File
@@ -38,12 +38,12 @@ import org.archive.io.arc.ARCReaderFactory;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
/**
* Base class for 'self tests', integrations tests formatted as unit
@@ -182,7 +182,7 @@ public abstract class SelfTestBase extends TmpDirTestCase {
protected void startHttpServer() throws Exception {
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java
+7 -7
View File
@@ -48,12 +48,12 @@ public class StatisticsSelfTest extends SelfTestBase {
StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker();
assertNotNull(stats);
assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
assertEquals(7501, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(2133, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(5368, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
}
@@ -66,17 +66,17 @@ public class StatisticsSelfTest extends SelfTestBase {
sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html");
assertNotNull(sourceStats);
assertEquals(4, sourceStats.keySet().size());
assertEquals(2942l, (long) sourceStats.get("novel"));
assertEquals(2133l, (long) sourceStats.get("novel"));
assertEquals(3l, (long) sourceStats.get("novelCount"));
assertEquals(2942l, (long) sourceStats.get("warcNovelContentBytes"));
assertEquals(2133l, (long) sourceStats.get("warcNovelContentBytes"));
assertEquals(3l, (long) sourceStats.get("warcNovelUrls"));
sourceStats = stats.getSourceStats("http://localhost:7777/b.html");
assertNotNull(sourceStats);
assertEquals(4, sourceStats.keySet().size());
assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
assertEquals(5368l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
assertEquals(11l, (long) sourceStats.get("novelCount"));
assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
assertEquals(5368l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
assertEquals(10l, (long) sourceStats.get("warcNovelUrls"));
}
engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java
+11 -11
View File
@@ -20,14 +20,14 @@
package org.archive.crawler.selftest;
import org.archive.util.ArchiveUtils;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.ServletHolder;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
/**
* @author pjack
@@ -54,17 +54,17 @@ public class UserAgentSelfTest extends SelfTestBase {
protected void startHttpServer() throws Exception {
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
ServletHandler servletHandler = new ServletHandler();
ServletHandler servletHandler = new ServletHandler();
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {
handlers.setHandlers(new Handler[] {
rhandler,
servletHandler,
new DefaultHandler() });
engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java
+30 -32
View File
@@ -44,20 +44,20 @@ import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.NCSARequestLog;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.HandlerCollection;
import org.mortbay.jetty.handler.RequestLogHandler;
import org.mortbay.jetty.security.Authenticator;
import org.mortbay.jetty.security.Constraint;
import org.mortbay.jetty.security.ConstraintMapping;
import org.mortbay.jetty.security.FormAuthenticator;
import org.mortbay.jetty.security.HashUserRealm;
import org.mortbay.jetty.security.SecurityHandler;
import org.mortbay.jetty.servlet.HashSessionManager;
import org.mortbay.jetty.servlet.SessionHandler;
import org.eclipse.jetty.security.*;
import org.eclipse.jetty.security.authentication.FormAuthenticator;
import org.eclipse.jetty.server.NCSARequestLog;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.handler.RequestLogHandler;
import org.eclipse.jetty.server.session.DefaultSessionCache;
import org.eclipse.jetty.server.session.NullSessionDataStore;
import org.eclipse.jetty.server.session.SessionCache;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.security.Password;
/* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but
* the code is written, it's easier to run in eclipse, and no doubt tests
@@ -211,12 +211,9 @@ public class FormAuthTest extends TestCase {
public FormAuthTestHandler() {
super();
}
@Override
public void handle(String target, HttpServletRequest request,
HttpServletResponse response, int dispatch) throws IOException,
ServletException {
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
if (target.endsWith("/set-cookie")) {
response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value"));
}
@@ -247,16 +244,16 @@ public class FormAuthTest extends TestCase {
constraintMapping.setConstraint(constraint);
constraintMapping.setPathSpec("/auth/*");
SecurityHandler authWrapper = new SecurityHandler();
UserStore userStore = new UserStore();
userStore.addUser(login, new Password(password), new String[]{role});
HashLoginService loginService = new HashLoginService(realm);
loginService.setUserStore(userStore);
ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler();
authWrapper.setAuthenticator(authenticator);
authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping});
authWrapper.setUserRealm(new HashUserRealm(realm) {
{
put(login, password);
addUserToRole(login, role);
}
});
authWrapper.setLoginService(loginService);
return authWrapper;
}
@@ -265,7 +262,7 @@ public class FormAuthTest extends TestCase {
// server for form auth
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7779);
server.addConnector(sc);
@@ -277,16 +274,17 @@ public class FormAuthTest extends TestCase {
requestLogHandler.setRequestLog(requestLog);
handlers.addHandler(requestLogHandler);
FormAuthenticator formAuthenticatrix = new FormAuthenticator();
formAuthenticatrix.setLoginPage("/login.html");
FormAuthenticator formAuthenticatrix = new FormAuthenticator("/login.html", null, false);
SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix,
FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN,
FORM_AUTH_PASSWORD);
authWrapper.setHandler(handlers);
SessionHandler sessionHandler = new SessionHandler();
sessionHandler.setSessionManager(new HashSessionManager());
SessionCache cache = new DefaultSessionCache(sessionHandler);
cache.setSessionDataStore(new NullSessionDataStore());
sessionHandler.setSessionCache(cache);
sessionHandler.setHandler(authWrapper);
server.setHandler(sessionHandler);
engine/src/test/resources/log4j.xml
+1 -1
View File
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>
<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>
modules/pom.xml
+6 -14
View File
@@ -35,23 +35,15 @@
<version>1.6.3</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>6.1.26</version>
<scope>test</scope>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.4.19.v20190610</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.26</version>
<scope>test</scope>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
<version>9.4.19.v20190610</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.26</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.littleshoot</groupId>
<artifactId>littleproxy</artifactId>
modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java
+22 -18
View File
@@ -43,18 +43,17 @@ import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.archive.util.KeyTool;
import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.security.SslSocketConnector;
import org.mortbay.jetty.servlet.SessionHandler;
import org.mortbay.log.Log;
import com.google.common.io.Files;
import junit.extensions.TestSetup;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.ssl.SslContextFactory;
public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
@@ -64,9 +63,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
}
@Override
public void handle(String target, HttpServletRequest request,
HttpServletResponse response, int dispatch) throws IOException,
ServletException {
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
if (request.getParameter("name") != null) {
Cookie cookie = new javax.servlet.http.Cookie(request.getParameter("name"),
request.getParameter("value"));
@@ -114,7 +111,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
server.setHandler(new TestHandler());
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
@@ -133,11 +130,18 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
"-dname", "CN=127.0.0.1",
"-validity","3650"}); // 10 yr validity
SslSocketConnector ssc = new SslSocketConnector();
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD);
sslContextFactory.setKeyStorePath(keystoreFile.getPath());
HttpConfiguration httpsConfig = new HttpConfiguration();
httpsConfig.addCustomizer(new SecureRequestCustomizer());
ServerConnector ssc = new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(httpsConfig));
ssc.setHost("127.0.0.1");
ssc.setPort(7443);
ssc.setKeyPassword(KEYSTORE_PASSWORD);
ssc.setKeystore(keystoreFile.getPath());
server.addConnector(sc);
server.addConnector(ssc);
@@ -345,7 +349,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=example.com");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n"));
// check second fetch has expected cookie
curi = makeCrawlURI("http://example.com:7777/");
@@ -379,7 +383,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=.example.com");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=.example.com\r\n"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=.example.com\r\n"));
// check second fetch has expected cookie
curi = makeCrawlURI("http://example.com:7777/");
@@ -413,7 +417,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=somethingelse.com");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=somethingelse.com\r\n"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=somethingelse.com\r\n"));
// check fetch of original domain has no cookie
curi = makeCrawlURI("http://example.com:7777/");
@@ -434,7 +438,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=BAR.example.com");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=bar.example.com\r\n"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=bar.example.com\r\n"));
// check fetch of original domain has no cookie
curi = makeCrawlURI("http://foo.example.com:7777/");
@@ -463,7 +467,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=example.com");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n"));
curi = makeCrawlURI("http://FOO.example.com:7777/");
fetcher().process(curi);
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+44 -60
View File
@@ -30,27 +30,21 @@ import javax.servlet.http.HttpServletResponse;
import org.archive.modules.ProcessorTestBase;
import org.archive.util.KeyTool;
import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.NCSARequestLog;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Response;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.HandlerCollection;
import org.mortbay.jetty.handler.RequestLogHandler;
import org.mortbay.jetty.security.Authenticator;
import org.mortbay.jetty.security.BasicAuthenticator;
import org.mortbay.jetty.security.Constraint;
import org.mortbay.jetty.security.ConstraintMapping;
import org.mortbay.jetty.security.DigestAuthenticator;
import org.mortbay.jetty.security.HashUserRealm;
import org.mortbay.jetty.security.SecurityHandler;
import org.mortbay.jetty.security.SslSocketConnector;
import org.mortbay.jetty.servlet.SessionHandler;
import org.mortbay.log.Log;
import junit.extensions.TestSetup;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.security.*;
import org.eclipse.jetty.security.authentication.BasicAuthenticator;
import org.eclipse.jetty.security.authentication.DigestAuthenticator;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.security.Password;
import org.eclipse.jetty.util.ssl.SslContextFactory;
public class FetchHTTPTest extends ProcessorTestBase {
@@ -125,12 +119,13 @@ public class FetchHTTPTest extends ProcessorTestBase {
public TestHandler() {
super();
}
@Override
public void handle(String target, HttpServletRequest request,
HttpServletResponse response, int dispatch) throws IOException,
ServletException {
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
// echo the remote host back to the client so tests can reference it
response.setHeader("Client-Host", request.getRemoteHost());
if (target.endsWith("/set-cookie")) {
response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value"));
}
@@ -221,9 +216,7 @@ public class FetchHTTPTest extends ProcessorTestBase {
}
protected static Map<Integer, Server> httpServers;
protected static Request lastRequest = null;
protected static Response lastResponse = null;
protected static SecurityHandler makeAuthWrapper(Authenticator authenticator,
final String role, String realm, final String login,
final String password) {
@@ -235,16 +228,15 @@ public class FetchHTTPTest extends ProcessorTestBase {
constraintMapping.setConstraint(constraint);
constraintMapping.setPathSpec("/auth/*");
SecurityHandler authWrapper = new SecurityHandler();
ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler();
authWrapper.setAuthenticator(authenticator);
authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping});
authWrapper.setUserRealm(new HashUserRealm(realm) {
{
put(login, password);
addUserToRole(login, role);
}
});
UserStore userStore = new UserStore();
userStore.addUser(login, new Password(password), new String[] {role});
HashLoginService loginService = new HashLoginService(realm);
loginService.setUserStore(userStore);
authWrapper.setLoginService(loginService);
return authWrapper;
}
@@ -259,24 +251,10 @@ public class FetchHTTPTest extends ProcessorTestBase {
HashMap<Integer, Server> servers = new HashMap<Integer,Server>();
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new TestHandler());
RequestLogHandler requestLogHandler = new RequestLogHandler();
NCSARequestLog requestLog = new NCSARequestLog() {
@Override
public void log(Request request, Response response) {
super.log(request, response);
lastRequest = request;
lastResponse = response;
}
};
requestLogHandler.setRequestLog(requestLog);
handlers.addHandler(requestLogHandler);
// server for basic auth
Server server = new Server();
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
@@ -284,6 +262,8 @@ public class FetchHTTPTest extends ProcessorTestBase {
SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(),
BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN,
BASIC_AUTH_PASSWORD);
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new TestHandler());
authWrapper.setHandler(handlers);
server.setHandler(authWrapper);
@@ -299,16 +279,23 @@ public class FetchHTTPTest extends ProcessorTestBase {
"-storepass", KEYSTORE_PASSWORD,
"-keypass", KEYSTORE_PASSWORD,
"-alias", "jetty",
"-genkey",
"-genkey",
"-keyalg", "RSA",
"-dname", "CN=127.0.0.1",
"-validity","3650"}); // 10 yr validity
SslSocketConnector ssc = new SslSocketConnector();
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD);
sslContextFactory.setKeyStorePath(keystoreFile.getPath());
HttpConfiguration httpsConfig = new HttpConfiguration();
httpsConfig.addCustomizer(new SecureRequestCustomizer());
ServerConnector ssc = new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(httpsConfig));
ssc.setHost("127.0.0.1");
ssc.setPort(7443);
ssc.setKeyPassword(KEYSTORE_PASSWORD);
ssc.setKeystore(keystoreFile.getPath());
server.addConnector(ssc);
@@ -317,7 +304,7 @@ public class FetchHTTPTest extends ProcessorTestBase {
// server for digest auth
server = new Server();
sc = new SocketConnector();
sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7778);
server.addConnector(sc);
@@ -325,7 +312,9 @@ public class FetchHTTPTest extends ProcessorTestBase {
authWrapper = makeAuthWrapper(new DigestAuthenticator(),
DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN,
DIGEST_AUTH_PASSWORD);
authWrapper.setHandler(handlers);
HandlerCollection handlers2 = new HandlerCollection();
handlers2.addHandler(new TestHandler());
authWrapper.setHandler(handlers2);
server.setHandler(authWrapper);
server.start();
@@ -333,7 +322,7 @@ public class FetchHTTPTest extends ProcessorTestBase {
return servers;
}
protected static void ensureHttpServers() throws Exception {
if (httpServers == null) {
httpServers = startHttpServers();
@@ -362,9 +351,4 @@ public class FetchHTTPTest extends ProcessorTestBase {
}
};
}
public static Request getLastRequest() {
return lastRequest;
}
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+7 -5
View File
@@ -157,7 +157,7 @@ public class FetchHTTPTests extends ProcessorTestBase {
assertEquals(DEFAULT_PAYLOAD_STRING, curi.getRecorder().getContentReplayCharSequence().toString());
if (!exclusions.contains("httpBindAddress")) {
assertEquals("127.0.0.1", FetchHTTPTest.getLastRequest().getRemoteAddr());
assertTrue(rawResponseString(curi).contains("Client-Host: 127.0.0.1\r\n"));
}
if (!exclusions.contains("nonFatalFailuresIsEmpty")) {
@@ -262,7 +262,7 @@ public class FetchHTTPTests extends ProcessorTestBase {
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate"));
assertEquals("basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate"));
assertTrue(curi.getCredentials().contains(basicAuthCredential));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic"));
@@ -406,8 +406,8 @@ public class FetchHTTPTests extends ProcessorTestBase {
fetcher().process(curi);
// the client bind address isn't recorded anywhere in heritrix as
// far as i can tell, so we get it this way...
assertEquals(addr, FetchHTTPTest.getLastRequest().getRemoteAddr());
// far as i can tell, so we get the server to echo it back to us...
assertTrue(rawResponseString(curi).contains("Client-Host: " + addr + "\r\n"));
runDefaultChecks(curi, "httpBindAddress");
}
@@ -780,7 +780,9 @@ public class FetchHTTPTests extends ProcessorTestBase {
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
assertTrue(httpRequestString(curi).startsWith("GET /99% HTTP/1.0\r\n"));
runDefaultChecks(curi, "requestLine");
// jetty 9 rejects requests with paths like this with 400 Bad Request
// so we can't run these checks anymore
//runDefaultChecks(curi, "requestLine");
}
public void testTwoQuestionMarks() throws Exception {
modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java
+7 -13
View File
@@ -70,11 +70,11 @@ import org.archive.spring.ConfigPath;
import org.archive.util.Base32;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.HandlerCollection;
import org.mortbay.jetty.servlet.SessionHandler;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.session.SessionHandler;
public class ContentDigestHistoryTest extends TmpDirTestCase {
@@ -237,7 +237,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
fetcher.process(curi1);
assertEquals(200, curi1.getFetchStatus());
assertEquals(141, curi1.getContentSize());
assertEquals(expectedDigest, curi1.getContentDigestSchemeString());
assertFalse(curi1.hasContentDigestHistory());
@@ -260,7 +259,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
fetcher.process(curi2);
assertEquals(200, curi1.getFetchStatus());
assertEquals(141, curi1.getContentSize());
assertEquals(expectedDigest, curi1.getContentDigestSchemeString());
assertFalse(curi2.hasContentDigestHistory());
@@ -305,7 +303,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
assertTrue(recordIterator.hasNext());
record = recordIterator.next();
assertEquals(WARCRecordType.response.toString(), record.getHeader().getHeaderValue(HEADER_KEY_TYPE));
assertEquals("141", record.getHeader().getHeaderValue(CONTENT_LENGTH));
assertEquals(expectedDigest, record.getHeader().getHeaderValue(HEADER_KEY_PAYLOAD_DIGEST));
assertEquals(curi1.getUURI().toString(), record.getHeader().getHeaderValue(HEADER_KEY_URI));
assertEquals(payloadRecordIdWithBrackets, record.getHeader().getHeaderValue(HEADER_KEY_ID));
@@ -374,10 +371,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new SessionHandler(){
@Override
public void handle(String target, HttpServletRequest request,
HttpServletResponse response, int dispatch) throws IOException,
ServletException {
public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
@@ -388,7 +382,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
Server server = new Server();
server.setHandler(handlers);
SocketConnector sc = new SocketConnector();
ServerConnector sc = new ServerConnector(server);
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
modules/src/test/resources/log4j.xml
+1 -1
View File
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>
<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>
pom.xml
+2 -2
View File
@@ -373,8 +373,8 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>