diff --git a/.travis.yml b/.travis.yml index 43c4eec6..e234b918 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,14 @@ + sudo: false language: java - -jdk: - - oraclejdk8 - - openjdk7 - - openjdk8 - matrix: - allow_failures: - - jdk: openjdk7 - + include: + - jdk: oraclejdk8 + dist: trusty + - jdk: openjdk8 + - jdk: openjdk11 + before_install: - "export JAVA_OPTS=-Xmx1500m" - "echo JAVA_OPTS=$JAVA_OPTS" @@ -18,6 +16,8 @@ before_install: - "echo MAVEN_OPTS=$MAVEN_OPTS" - "export _JAVA_OPTIONS=-Xmx1500m" - "echo _JAVA_OPTIONS=$_JAVA_OPTIONS" + +install: mvn dependency:resolve -B -V cache: directories: @@ -25,7 +25,6 @@ cache: script: - travis_wait 30 mvn install - - cd contrib && mvn install after_failure: - cat */target/surefire-reports/*.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..90edcee4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,11 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 5f307433..2dac370f 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,37 @@ -Readme for Heritrix -==================== +# Heritrix +[![Build Status](https://travis-ci.org/internetarchive/heritrix3.svg?branch=master)](https://travis-ci.org/internetarchive/heritrix3) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix) +[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine) +[![LICENSE](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](./LICENSE) -1. Introduction -2. Crawl Operators! -3. Getting Started -4. Developer Documentation -5. Release History -6. License +## Introduction + +Heritrix is the Internet Archive's open-source, extensible, web-scale, archival-quality web crawler project. Heritrix (sometimes spelled heretrix, or misspelled or missaid as heratrix/heritix/heretix/heratix) is an archaic word for heiress (woman who inherits). Since our crawler seeks to collect and preserve the digital artifacts of our culture for the benefit of future researchers and generations, this name seemed apt. + +## Crawl Operators! + +Heritrix is designed to respect the [`robots.txt`](http://www.robotstxt.org/wc/robots.html) exclusion directives and [META robots tags](http://www.robotstxt.org/wc/exclusion.html#meta). Please consider the +load your crawl will place on seed sites and set politeness policies accordingly. Also, always identify your crawl with contact information in the `User-Agent` so sites that may be adversely affected by your crawl can contact you or adapt their server behavior accordingly. + +## Getting Started + +- [User Manual](https://github.com/internetarchive/heritrix3/wiki) + +## Developer Documentation + +- [Developer Manual](http://crawler.archive.org/articles/developer_manual/index.html) +- [REST API documentation](https://heritrix.readthedocs.io/en/latest/api.html) +- JavaDoc: [engine](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine), [modules](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-modules), [commons](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-commons), [contrib](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-contrib) -## 1. Introduction +## Latest Releases -Heritrix is the Internet Archive's open-source, extensible, web-scale, -archival-quality web crawler project. Heritrix (sometimes spelled heretrix, or -misspelled or missaid as heratrix/heritix/heretix/heratix) is an archaic word -for heiress (woman who inherits). Since our crawler seeks to collect and -preserve the digital artifacts of our culture for the benefit of future -researchers and generations, this name seemed apt. +Information about releases can be found [here](https://github.com/internetarchive/heritrix3/wiki#latest-releases). +## License -## 2. Crawl Operators! +Heritrix is free software; you can redistribute it and/or modify it under the terms of the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) -Heritrix is designed to respect the robots.txt - exclusion directives and META robots -tags . Please consider the -load your crawl will place on seed sites and set politeness policies -accordingly. Also, always identify your crawl with contact information in the -User-Agent so sites that may be adversely affected by your crawl can contact -you or adapt their server behavior accordingly. - - -## 3. Getting Started - -See the User Manual, available from - - -## 4. Developer Documentation - -See . -For REST API documentation, see -and for JavaDoc see (n.b. Javadoc currently out of date). - - -## 5. Latest Releases - -Information about releases can be found at - - -## 6. License - -Heritrix is free software; you can redistribute it and/or modify it -under the terms of the Apache License, Version 2.0: - - http://www.apache.org/licenses/LICENSE-2.0 - -Some individual source code files are subject to or offered under other -licenses. See the included LICENSE.txt file for more information. - -Heritrix is distributed with the libraries it depends upon. The -libraries can be found under the 'lib' directory, and are used under -the terms of their respective licenses, which are included alongside -the libraries in the 'lib' directory. +Some individual source code files are subject to or offered under other licenses. See the included [`LICENSE.txt`](./LICENSE) file for more information. +Heritrix is distributed with the libraries it depends upon. The libraries can be found under the `lib` directory in the release distribution, and are used under the terms of their respective licenses, which are included alongside the libraries in the `lib` directory. diff --git a/commons/pom.xml b/commons/pom.xml index 3a9da7dc..3a121b02 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -221,7 +221,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.9 + 2.22.2 + + jdk.tools + jdk.tools + diff --git a/contrib/src/main/resources/log4j.xml b/contrib/src/main/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/contrib/src/main/resources/log4j.xml +++ b/contrib/src/main/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/contrib/src/test/resources/log4j.xml b/contrib/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/contrib/src/test/resources/log4j.xml +++ b/contrib/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/dist/pom.xml b/dist/pom.xml index 64c931bd..9a7ddc07 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -25,7 +25,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.9 + 2.22.2 true diff --git a/dist/src/main/conf/logging.properties b/dist/src/main/conf/logging.properties index f8372396..094cb176 100644 --- a/dist/src/main/conf/logging.properties +++ b/dist/src/main/conf/logging.properties @@ -5,7 +5,7 @@ # ...and even less from the too-chatty-with-WARNINGs HttpClient library... org.apache.commons.httpclient.level = SEVERE org.restlet.Component.LogFilter.level = SEVERE -org.mortbay.log.level = SEVERE +org.eclipse.jetty.log.level = SEVERE # ...but INFO for our classes, which reserve FINE/FINER/FINEST for bulk/trivia... org.archive.level = INFO diff --git a/dist/src/test/resources/log4j.xml b/dist/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/dist/src/test/resources/log4j.xml +++ b/dist/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/engine/pom.xml b/engine/pom.xml index 14410e9e..9051e9e0 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -25,50 +25,68 @@ ${project.version} compile + - org.mortbay.jetty - jetty - 6.1.26 - compile + org.eclipse.jetty + jetty-server + 9.4.19.v20190610 - org.mortbay.jetty + org.eclipse.jetty jetty-util - 6.1.26 - compile + 9.4.19.v20190610 - org.mortbay.jetty - jetty-sslengine - 6.1.26 - compile + org.eclipse.jetty + jetty-servlet + 9.4.19.v20190610 + test + + + org.restlet.jse + org.restlet + 2.4.0 + + + org.restlet.jse + org.restlet.ext.jetty + 2.4.0 + + + org.eclipse.jetty + jetty-client + + - org.mortbay.jetty - jetty-ajp - 6.1.26 - compile + org.restlet.jse + org.restlet.ext.xml + 2.4.0 - org.restlet - org.restlet - 1.1.10 - - - com.noelios.restlet - com.noelios.restlet - 1.1.10 - - - com.noelios.restlet - com.noelios.restlet.ext.jetty - 1.1.10 + + org.restlet.jse + org.restlet.ext.crypto + 2.4.0 joda-time joda-time 1.6 + + + javax.xml.bind + jaxb-api + 2.3.1 + + + org.glassfish.jaxb + jaxb-runtime + 2.3.1 + runtime + org.littleshoot littleproxy @@ -99,7 +117,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.9 + 2.22.2 **/TestAll.java diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 11495072..dd9e208e 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -33,12 +33,7 @@ import java.net.UnknownHostException; import java.security.KeyStore; import java.security.MessageDigest; import java.security.cert.Certificate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; import java.util.logging.LogManager; import java.util.logging.Logger; @@ -57,10 +52,11 @@ import org.archive.crawler.restlet.RateLimitGuard; import org.archive.util.ArchiveUtils; import org.archive.util.KeyTool; import org.restlet.Component; -import org.restlet.Guard; import org.restlet.Server; import org.restlet.data.ChallengeScheme; import org.restlet.data.Protocol; +import org.restlet.security.ChallengeAuthenticator; +import org.restlet.security.MapVerifier; /** @@ -212,7 +208,7 @@ public class Heritrix { "mailto, clsid, res, file, rtsp, about"); } - String maxFormSize = "org.mortbay.jetty.Request.maxFormContentSize"; + String maxFormSize = "org.eclipse.jetty.server.Request.maxFormContentSize"; if (System.getProperty(maxFormSize) == null) { System.setProperty(maxFormSize, "52428800"); } @@ -334,24 +330,30 @@ public class Heritrix { try { engine = new Engine(jobsDir); component = new Component(); - + if(bindHosts.isEmpty()) { // listen all addresses - setupServer(port, null, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, null, keystorePath, keystorePassword, keyPassword); } else { // bind only to declared addresses, or just 'localhost' for(String address : bindHosts) { - setupServer(port, address, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, address, keystorePath, keystorePassword, keyPassword); } } component.getClients().add(Protocol.FILE); - component.getClients().add(Protocol.CLAP); - Guard guard = new RateLimitGuard(null, - ChallengeScheme.HTTP_DIGEST, "Authentication Required"); - guard.getSecrets().put(authLogin, authPassword.toCharArray()); - component.getDefaultHost().attach(guard); + component.getClients().add(Protocol.CLAP); + + MapVerifier verifier = new MapVerifier(); + verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); + + RateLimitGuard guard = new RateLimitGuard(component.getContext().createChildContext(), + "Authentication Required", UUID.randomUUID().toString()); + guard.setWrappedVerifier(verifier); guard.setNext(new EngineApplication(engine)); + + component.getDefaultHost().attach(guard); component.start(); + startupOut.println("engine listening at port "+port); startupOut.println("operator login set per " + ((aOption.startsWith("@")) ? "file "+aOption : "command-line")); @@ -457,16 +459,16 @@ public class Heritrix { /** * Create an HTTPS restlet Server instance matching the given parameters. - * + * + * @param component * @param port * @param address * @param keystorePath * @param keystorePassword * @param keyPassword */ - protected void setupServer(int port, String address, String keystorePath, String keystorePassword, String keyPassword) { - Server server = new Server(Protocol.HTTPS,address,port,null); - component.getServers().add(server); + protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) { + Server server = component.getServers().add(Protocol.HTTPS, address, port); server.getContext().getParameters().add("keystorePath", keystorePath); server.getContext().getParameters().add("keystorePassword", keystorePassword); server.getContext().getParameters().add("keyPassword", keyPassword); diff --git a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java index d187bfc6..75875184 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java @@ -18,7 +18,9 @@ */ package org.archive.crawler.frontier; +import org.apache.commons.lang.StringUtils; import org.archive.modules.CrawlURI; +import org.archive.net.UURI; import org.archive.net.PublicSuffixes; /** @@ -32,9 +34,34 @@ public class AssignmentLevelSurtQueueAssignmentPolicy extends private static final long serialVersionUID = -1533545293624791702L; @Override - public String getClassKey(CrawlURI cauri) { - String candidate = super.getClassKey(cauri); - candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + public String getClassKey(CrawlURI curi) { + if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { + return curi.getClassKey(); + } + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = super.getClassKey(curi); + candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + + if(!StringUtils.isEmpty(getForceQueueAssignment())) { + candidate = getForceQueueAssignment(); + } + + // all whois urls in the same queue + if (curi.getUURI().getScheme().equals("whois")) { + return "whois..."; + } + + if(StringUtils.isEmpty(candidate)) { + return DEFAULT_CLASS_KEY; + } + if(getParallelQueues()>1) { + int subqueue = getSubqueue(basis,getParallelQueues()); + if (subqueue>0) { + candidate += "+"+subqueue; + } + } + return candidate; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 8001540d..3212c4f0 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -90,9 +90,12 @@ implements if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { return curi.getClassKey(); } + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = getCoreKey(basis); if(!StringUtils.isEmpty(getForceQueueAssignment())) { - return getForceQueueAssignment(); + candidate = getForceQueueAssignment(); } // all whois urls in the same queue @@ -100,9 +103,6 @@ implements return "whois..."; } - UURI basis = curi.getPolicyBasisUURI(); - String candidate = getCoreKey(basis); - if(StringUtils.isEmpty(candidate)) { return DEFAULT_CLASS_KEY; } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index d7630227..171bb5d0 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -348,7 +348,7 @@ public class QuotaEnforcer extends Processor { * being force-retired (if the Frontier supports this). Note that if your * queues combine URIs that are different with regard to the quota category, * the retirement may hold back URIs not in the same quota category. Default - * is false. + * is true. */ { setForceRetire(true); diff --git a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java index 1ebc80ef..72ae434b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java @@ -19,55 +19,14 @@ package org.archive.crawler.restlet; -import java.util.List; - -import org.restlet.Context; -import org.restlet.data.MediaType; -import org.restlet.data.Preference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Resource; -import org.restlet.resource.Variant; +import org.restlet.resource.ServerResource; /** * Abstract {@code Resource} with common shared functionality. * * @author nlevitt */ -public abstract class BaseResource extends Resource { - - public BaseResource(Context ctx, Request req, Response res) { - super(ctx, req, res); - } - - /** - * If client can accept text/html, always prefer it. WebKit-based browsers - * claim to want application/xml, but we don't want to give it to them. See - * https://webarchive.jira.com/browse/HER-1603 - */ - public Variant getPreferredVariant() { - boolean addExplicitTextHtmlPreference = false; - - for (Preference mediaTypePreference: getRequest().getClientInfo().getAcceptedMediaTypes()) { - if (mediaTypePreference.getMetadata().equals(MediaType.TEXT_HTML)) { - mediaTypePreference.setQuality(Float.MAX_VALUE); - addExplicitTextHtmlPreference = false; - break; - } else if (mediaTypePreference.getMetadata().includes(MediaType.TEXT_HTML)) { - addExplicitTextHtmlPreference = true; - } - } - - if (addExplicitTextHtmlPreference) { - List> acceptedMediaTypes = getRequest().getClientInfo().getAcceptedMediaTypes(); - acceptedMediaTypes.add(new Preference(MediaType.TEXT_HTML, Float.MAX_VALUE)); - getRequest().getClientInfo().setAcceptedMediaTypes(acceptedMediaTypes); - } - - - return super.getPreferredVariant(); - } - +public abstract class BaseResource extends ServerResource { protected String getStaticRef(String resource) { String rootRef = getRequest().getRootRef().toString(); return rootRef + "/engine/static/" + resource; diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index b09d70af..5ae0c897 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -19,16 +19,12 @@ package org.archive.crawler.restlet; -import java.io.File; import java.io.IOException; -import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.net.URLDecoder; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Set; @@ -36,18 +32,18 @@ import org.apache.commons.lang.StringUtils; import org.archive.crawler.restlet.models.BeansModel; import org.archive.crawler.restlet.models.ViewModel; import org.archive.spring.PathSharingContext; -import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import org.springframework.beans.BeanWrapperImpl; import org.springframework.beans.BeansException; @@ -69,12 +65,12 @@ public class BeanBrowseResource extends JobRelatedResource { protected PathSharingContext appCtx; protected String beanPath; private Configuration _templateConfiguration; - - public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); - setModifiable(true); // accept POSTs appCtx = cj.getJobContext(); beanPath = (String)req.getAttributes().get("beanPath"); if (beanPath!=null) { @@ -99,13 +95,14 @@ public class BeanBrowseResource extends JobRelatedResource { return _templateConfiguration; } - public void acceptRepresentation(Representation entity) throws ResourceException { + @Override + protected Representation post(Representation entity, Variant variant) throws ResourceException { if (appCtx == null) { throw new ResourceException(404); } // copy op? - Form form = getRequest().getEntityAsForm(); + Form form = new Form(entity); beanPath = form.getFirstValue("beanPath"); String newVal = form.getFirstValue("newVal"); @@ -122,6 +119,7 @@ public class BeanBrowseResource extends JobRelatedResource { ref.setPath(getBeansRefPath()); ref.addSegment(beanPath); getResponse().redirectSeeOther(ref); + return new EmptyRepresentation(); } public String getBeansRefPath() { @@ -137,7 +135,8 @@ public class BeanBrowseResource extends JobRelatedResource { return path; } - public Representation represent(Variant variant) throws ResourceException { + @Override + public Representation get(Variant variant) throws ResourceException { if (appCtx == null) { throw new ResourceException(404); } @@ -209,8 +208,8 @@ public class BeanBrowseResource extends JobRelatedResource { for(String name: appCtx.getBeanDefinitionNames()) { addPresentableNestedNames(nestedNames, appCtx.getBean(name), alreadyWritten); } - - return new BeansModel(cj.getShortName(), + + return new BeansModel(cj.getShortName(), new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef().toString(), beanPath, bean, diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index a4ea7996..232dbdd2 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -30,8 +30,8 @@ import org.apache.commons.lang.StringEscapeUtils; import org.restlet.data.CharacterSet; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.resource.CharacterRepresentation; -import org.restlet.resource.FileRepresentation; +import org.restlet.representation.CharacterRepresentation; +import org.restlet.representation.FileRepresentation; /** * Representation wrapping a FileRepresentation, displaying its contents @@ -40,9 +40,9 @@ import org.restlet.resource.FileRepresentation; * @author gojomo */ public class EditRepresentation extends CharacterRepresentation { - protected FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; protected EnhDirectoryResource dirResource; - + public EditRepresentation(FileRepresentation representation, EnhDirectoryResource resource) { super(MediaType.TEXT_HTML); fileRepresentation = representation; diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java index 6c7793f7..a82c3026 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java @@ -26,19 +26,19 @@ import java.io.StringWriter; import org.archive.crawler.framework.Engine; import org.archive.util.TextUtils; import org.restlet.Application; -import org.restlet.Directory; -import org.restlet.Redirector; import org.restlet.Restlet; -import org.restlet.Router; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.Status; -import org.restlet.resource.Representation; -import org.restlet.resource.StringRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; +import org.restlet.resource.Directory; +import org.restlet.routing.Redirector; +import org.restlet.routing.Router; +import org.restlet.routing.Template; import org.restlet.service.StatusService; -import org.restlet.util.Template; /** * Restlet Application for a Heritrix crawl 'Engine', which is aware of @@ -56,7 +56,8 @@ public class EngineApplication extends Application { setStatusService(new EngineStatusService()); } - public synchronized Restlet createRoot() { + @Override + public Restlet createInboundRoot() { Router router = new Router(getContext()); router.attach("/",new Redirector(null,"/engine",Redirector.MODE_CLIENT_TEMPORARY)); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index b18d9f3d..dabd7f3e 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -30,21 +30,24 @@ import org.archive.crawler.framework.Engine; import org.archive.crawler.restlet.models.EngineModel; import org.archive.crawler.restlet.models.ViewModel; import org.restlet.Context; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; +import static org.restlet.data.MediaType.APPLICATION_XML; + /** * Restlet Resource representing an Engine that may be used * to assemble, launch, monitor, and manage crawls. @@ -56,11 +59,12 @@ import freemarker.template.TemplateException; public class EngineResource extends BaseResource { private Configuration _templateConfiguration; - public EngineResource(Context ctx, Request req, Response res) { - super(ctx, req, res); - setModifiable(true); + + @Override + public void init(Context ctx, Request req, Response res) { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); - getVariants().add(new Variant(MediaType.APPLICATION_XML)); + getVariants().add(new Variant(APPLICATION_XML)); Configuration tmpltCfg = new Configuration(); tmpltCfg.setClassForTemplateLoading(this.getClass(),""); @@ -74,10 +78,12 @@ public class EngineResource extends BaseResource { public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public Representation represent(Variant variant) throws ResourceException { + + @Override + protected Representation get(Variant variant) throws ResourceException { Representation representation; - if (variant.getMediaType() == MediaType.APPLICATION_XML) { - representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + if (variant.getMediaType() == APPLICATION_XML) { + representation = new WriterRepresentation(APPLICATION_XML) { public void write(Writer writer) throws IOException { XmlMarshaller.marshalDocument(writer, "engine", makeDataModel()); } @@ -93,10 +99,10 @@ public class EngineResource extends BaseResource { representation.setCharacterSet(CharacterSet.UTF_8); return representation; } - + @Override - public void acceptRepresentation(Representation entity) throws ResourceException { - Form form = getRequest().getEntityAsForm(); + protected Representation post(Representation entity, Variant variant) throws ResourceException { + Form form = new Form(entity); String action = form.getFirstValue("action"); if("rescan".equals(action)) { getEngine().findJobConfigs(); @@ -182,6 +188,7 @@ public class EngineResource extends BaseResource { } // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + return new EmptyRepresentation(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java index ea7bb79d..bb26b16b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java @@ -26,10 +26,12 @@ import java.util.logging.Level; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.restlet.Context; -import org.restlet.Handler; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.data.Status; +import org.restlet.resource.Directory; +import org.restlet.resource.ServerResource; /** * Enhanced version of Restlet Directory, which allows the local @@ -39,11 +41,11 @@ import org.restlet.data.Response; * * @author gojomo */ -public abstract class EnhDirectory extends org.restlet.Directory { +public abstract class EnhDirectory extends Directory { protected IOFileFilter editFilter = FileFilterUtils.falseFileFilter(); protected IOFileFilter pageFilter = FileFilterUtils.falseFileFilter(); protected IOFileFilter tailFilter = FileFilterUtils.falseFileFilter(); - + public EnhDirectory(Context context, Reference rootLocalReference) { super(context, rootLocalReference); // TODO Auto-generated constructor stub @@ -55,21 +57,27 @@ public abstract class EnhDirectory extends org.restlet.Directory { } @Override - public Handler findTarget(Request request, Response response) { - Handler retVal; - synchronized(this) { + public void handle(Request request, Response response) { + synchronized (this) { Reference oldRef = getRootRef(); setRootRef(determineRootRef(request)); try { - retVal = new EnhDirectoryResource(this, request, response); - } catch (IOException ioe) { - getLogger().log(Level.WARNING, - "Unable to find the directory's resource", ioe); - retVal = null; + super.handle(request, response); + } finally { + setRootRef(oldRef); + } + + // XXX: FileRepresentation.isAvailable() returns false for empty files generating status 204 No Content + // which confuses browsers. Force it back it 200 OK. + if (response.getStatus() == Status.SUCCESS_NO_CONTENT) { + response.setStatus(Status.SUCCESS_OK); } - setRootRef(oldRef); } - return retVal; + } + + @Override + public ServerResource create(Request request, Response response) { + return new EnhDirectoryResource(); } protected abstract Reference determineRootRef(Request request); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java index 5f7c0b5e..ffebd7e6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java @@ -21,24 +21,22 @@ package org.archive.crawler.restlet; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; +import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import org.apache.commons.io.FileUtils; -import org.restlet.data.CharacterSet; -import org.restlet.data.Form; -import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.data.Status; -import org.restlet.resource.FileRepresentation; -import org.restlet.resource.Representation; +import org.restlet.data.*; +import org.restlet.engine.local.DirectoryServerResource; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.FileRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.Variant; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; - -import com.noelios.restlet.local.DirectoryResource; /** * Enhanced version of Restlet DirectoryResource, adding ability to @@ -46,20 +44,19 @@ import com.noelios.restlet.local.DirectoryResource; * * @author gojomo */ -public class EnhDirectoryResource extends DirectoryResource { - - public EnhDirectoryResource(EnhDirectory directory, Request request, Response response) throws IOException { - super(directory, request, response); - } - - /** +public class EnhDirectoryResource extends DirectoryServerResource { + /** * Add EditRepresentation as a variant when appropriate. * - * @see com.noelios.restlet.local.DirectoryResource#getVariants() + * @see org.restlet.engine.local.DirectoryServerResource#getVariants() */ @Override public List getVariants() { - List variants = super.getVariants(); + List superVariants = super.getVariants(); + if (superVariants == null) { + return null; // PUT and DELETE return no content + } + List variants = new LinkedList<>(superVariants); Form f = getRequest().getResourceRef().getQueryAsForm(); String format = f.getFirstValue("format"); if("textedit".equals(format)) { @@ -73,7 +70,11 @@ public class EnhDirectoryResource extends DirectoryResource { } catch (Exception e) { throw new RuntimeException(e); } - variants = super.getVariants(); + superVariants = super.getVariants(); + if (superVariants == null) { + return null; + } + variants = new LinkedList<>(superVariants); } // wrap FileRepresentations in EditRepresentations ListIterator iter = variants.listIterator(); @@ -119,18 +120,18 @@ public class EnhDirectoryResource extends DirectoryResource { } protected EnhDirectory getEnhDirectory() { - return (EnhDirectory)getDirectory(); + return (EnhDirectory) getDirectory(); } - /** + /** * Accept a POST used to edit or create a file. * - * @see org.restlet.resource.Resource#acceptRepresentation(org.restlet.resource.Representation) + * @see org.restlet.resource.ServerResource#post(Representation, Variant) */ - public void acceptRepresentation(Representation entity) - throws ResourceException { + @Override + protected Representation post(Representation entity, Variant variant) throws ResourceException { // TODO: only allowPost on valid targets - Form form = getRequest().getEntityAsForm(); + Form form = new Form(entity); String newContents = form.getFirstValue("contents"); EditRepresentation er; try { @@ -152,6 +153,30 @@ public class EnhDirectoryResource extends DirectoryResource { Reference ref = getRequest().getOriginalRef().clone(); /// ref.setQuery(null); getResponse().redirectSeeOther(ref); - + return new EmptyRepresentation(); + } + + /* + * XXX: We override Restlet's default PUT behaviour (see FileClientHelper.handleFilePut) as it unhelpfully changes + * the file extension based on the content-type and there's no apparent way to disable that. + */ + @Override + public Representation put(Representation entity) throws ResourceException { + File file = new File(URI.create(getTargetUri())); + if (getTargetUri().endsWith("/") || file.isDirectory()) { + return super.put(entity); + } + boolean created = !file.exists(); + try (FileOutputStream out = new FileOutputStream(file)) { + entity.write(out); + } catch (FileNotFoundException e) { + throw new ResourceException(Status.CLIENT_ERROR_NOT_FOUND, e); + } catch (IOException e) { + throw new ResourceException(Status.SERVER_ERROR_INTERNAL, e); + } + if (created) { + getResponse().setStatus(Status.SUCCESS_CREATED); + } + return new EmptyRepresentation(); } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/Flash.java b/engine/src/main/java/org/archive/crawler/restlet/Flash.java index a43d6c9e..44140866 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/Flash.java +++ b/engine/src/main/java/org/archive/crawler/restlet/Flash.java @@ -29,10 +29,10 @@ import java.util.Map.Entry; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.RandomUtils; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.Cookie; import org.restlet.data.CookieSetting; -import org.restlet.data.Request; -import org.restlet.data.Response; import org.restlet.util.Series; /** diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index ae967a0a..4dfe548c 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -40,8 +40,8 @@ import org.archive.crawler.framework.Engine; import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.resource.ResourceException; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanWrapperImpl; @@ -58,18 +58,19 @@ public abstract class JobRelatedResource extends BaseResource { private final static Logger LOGGER = Logger.getLogger(JobRelatedResource.class.getName()); - protected CrawlJob cj; + protected CrawlJob cj; protected IdentityHashMap beanToNameMap; - - public JobRelatedResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) { + super.init(ctx, req, res); cj = getEngine().getJob((String)req.getAttributes().get("job")); - if(cj==null) { + if(cj == null) { throw new ResourceException(404); } } - + protected Engine getEngine() { return ((EngineApplication)getApplication()).getEngine(); } @@ -95,13 +96,13 @@ public abstract class JobRelatedResource extends BaseResource { return; } - - Reference baseRef = getRequest().getResourceRef().getBaseRef(); + + Reference baseRef = getRequest().getResourceRef().getBaseRef(); if (baseRef.getPath().endsWith("beans")) { baseRef.setPath(baseRef.getPath() + "/"); } - + if (getBeanToNameMap().containsKey(obj)) { // this object is itself a named bean Map bean = new LinkedHashMap(); diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index d3b0a295..5b4ab6c7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -38,12 +38,13 @@ import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.ObjectWrapper; @@ -68,10 +69,10 @@ public class JobResource extends BaseResource { protected CrawlJob cj; - public JobResource(Context ctx, Request req, Response res) + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); - setModifiable(true); + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); cj = getEngine().getJob( @@ -88,7 +89,9 @@ public class JobResource extends BaseResource { public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public Representation represent(Variant variant) throws ResourceException { + + @Override + public Representation get(Variant variant) throws ResourceException { if (cj == null) { throw new ResourceException(404); } @@ -184,19 +187,18 @@ public class JobResource extends BaseResource { } @Override - public void acceptRepresentation(Representation entity) + public Representation post(Representation entity, Variant variant) throws ResourceException { if (cj == null) { throw new ResourceException(404); } // copy op? - Form form = null; - form = getRequest().getEntityAsForm(); + Form form = new Form(entity); String copyTo = form.getFirstValue("copyTo"); if (copyTo != null) { copyJob(copyTo, "on".equals(form.getFirstValue("asProfile"))); - return; + return new EmptyRepresentation(); } AlertHandler.ensureStaticInitialization(); AlertThreadGroup.setThreadLogger(cj.getJobLogger()); @@ -241,6 +243,7 @@ public class JobResource extends BaseResource { // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + return new EmptyRepresentation(); } protected void copyJob(String copyTo, boolean asProfile) diff --git a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java index 363ee674..aabd0d46 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java @@ -38,8 +38,8 @@ import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.resource.CharacterRepresentation; -import org.restlet.resource.FileRepresentation; +import org.restlet.representation.CharacterRepresentation; +import org.restlet.representation.FileRepresentation; /** * Representation wrapping a FileRepresentation, displaying its contents @@ -50,7 +50,7 @@ import org.restlet.resource.FileRepresentation; public class PagedRepresentation extends CharacterRepresentation { // passed-in at construction /** wrapped FileRepresentation **/ - protected FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; /** wrapped EnhDirectoryResource; used to formulate self-links **/ protected EnhDirectoryResource dirResource; @@ -124,7 +124,7 @@ public class PagedRepresentation extends CharacterRepresentation { /** * Write the paged HTML. * - * @see org.restlet.resource.Representation#write(java.io.Writer) + * @see org.restlet.representation.Representation#write(java.io.Writer) */ @Override public void write(Writer writer) throws IOException { diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index d4d304ca..7d9cd703 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -18,39 +18,34 @@ */ package org.archive.crawler.restlet; -import java.util.Collection; +import org.restlet.Context; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.ext.crypto.DigestAuthenticator; + import java.util.logging.Logger; -import org.restlet.Context; -import org.restlet.Guard; -import org.restlet.data.ChallengeScheme; -import org.restlet.data.Request; - /** - * Guard that slows and logs failed authentication attempts, to make + * ChallengeAuthenticator that slows and logs failed authentication attempts, to make * brute-force guessing attacks less feasible. * * @author gojomo */ -public class RateLimitGuard extends Guard { +public class RateLimitGuard extends DigestAuthenticator { private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000; private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); protected long lastFailureTime = 0; - public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) throws IllegalArgumentException { - super(context, scheme, realm); - } - - public RateLimitGuard(Context context, String realm, Collection baseUris, String serverKey) { - super(context, realm, baseUris, serverKey); + public RateLimitGuard(Context context, String realm, String serverKey) throws IllegalArgumentException { + super(context, realm, serverKey); } @Override - public synchronized int authenticate(Request request) { - int retVal = super.authenticate(request); - if(retVal == AUTHENTICATION_INVALID) { + protected boolean authenticate(Request request, Response response) { + boolean succeeded = super.authenticate(request, response); + if (!succeeded) { logger.warning("authentication failure "+request); // wait until at least LAG has passed from last failure // holding object lock the whole time, so no other checks @@ -64,8 +59,8 @@ public class RateLimitGuard extends Guard { // ignore } } - lastFailureTime = now + sleepMs; + lastFailureTime = now + sleepMs; } - return retVal; + return succeeded; } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java index 3ccc8794..5468bf55 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java @@ -23,12 +23,12 @@ import java.io.File; import org.restlet.Context; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.StringRepresentation; -import org.restlet.resource.Variant; +import org.restlet.representation.Variant; /** * Restlet Resource which generates fresh reports and then redirects @@ -38,24 +38,26 @@ import org.restlet.resource.Variant; */ public class ReportGenResource extends JobRelatedResource { protected String reportClass; - - public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_PLAIN)); reportClass = (String)req.getAttributes().get("reportClass"); } - public Representation represent(Variant variant) throws ResourceException { + @Override + protected Representation get(Variant variant) throws ResourceException { // generate report if (cj == null || cj.getCrawlController() == null) { throw new ResourceException(500); } - File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); + File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); if (f==null) { throw new ResourceException(500); } // redirect - String relative = JobResource.getHrefPath(f,cj); + String relative = JobResource.getHrefPath(f, cj); if(relative!=null) { getResponse().redirectSeeOther("../"+relative+"?m="+f.lastModified()); return new StringRepresentation(""); diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 5a8bfa6b..58fee0f9 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -41,12 +41,12 @@ import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.ObjectWrapper; @@ -81,10 +81,10 @@ public class ScriptResource extends JobRelatedResource { protected String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0); private Configuration _templateConfiguration; - - public ScriptResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); - setModifiable(true); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); @@ -92,7 +92,7 @@ public class ScriptResource extends JobRelatedResource { tmpltCfg.setClassForTemplateLoading(this.getClass(),""); tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER); setTemplateConfiguration(tmpltCfg); - + scriptingConsole = new ScriptingConsole(cj); } public void setTemplateConfiguration(Configuration tmpltCfg) { @@ -105,8 +105,8 @@ public class ScriptResource extends JobRelatedResource { private ScriptingConsole scriptingConsole; @Override - public void acceptRepresentation(Representation entity) throws ResourceException { - Form form = getRequest().getEntityAsForm(); + public Representation post(Representation entity, Variant variant) throws ResourceException { + Form form = new Form(entity); chosenEngine = form.getFirstValue("engine"); String script = form.getFirstValue("script"); if(StringUtils.isBlank(script)) { @@ -121,10 +121,11 @@ public class ScriptResource extends JobRelatedResource { //TODO: log script, results somewhere; job log INFO? - getResponse().setEntity(represent()); + return get(variant); } - - public Representation represent(Variant variant) throws ResourceException { + + @Override + public Representation get(Variant variant) throws ResourceException { Representation representation; if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { diff --git a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java index 11098b38..176204c6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java +++ b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java @@ -36,7 +36,7 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; -import org.restlet.util.XmlWriter; +import org.restlet.ext.xml.XmlWriter; import org.xml.sax.SAXException; /** diff --git a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java index 138f47a1..7f7d9e6b 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java @@ -22,10 +22,10 @@ package org.archive.crawler.selftest; import java.io.IOException; import org.archive.crawler.framework.CrawlJob; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; /** @@ -85,7 +85,7 @@ public class CheckpointSelfTest extends SelfTestBase { private Server makeHttpServer(int port) throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost(HOST); sc.setPort(port); server.addConnector(sc); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java index 63d4b764..6e04f56a 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java @@ -19,19 +19,20 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; + import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; /** * Test form-based authentication @@ -60,17 +61,17 @@ public class FormAuthSelfTest protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java index 510fb839..332f6835 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java @@ -19,20 +19,20 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; + import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; - /** * Test form-based authentication * @@ -60,17 +60,17 @@ public class FormLoginSelfTest protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java index a0f68464..6ad5ebd2 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java @@ -24,17 +24,16 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.servlet.ServletHandler; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; /** * Test HTTP basic authentication @@ -73,27 +72,27 @@ public class HttpAuthSelfTest ConstraintMapping cm = new ConstraintMapping(); cm.setConstraint(constraint); cm.setPathSpec("/basic/*"); + + UserStore userStore = new UserStore(); + userStore.addUser("Mr. Happy Pants", new Password("xyzzy"), new String[]{"rule"}); + HashLoginService loginService = new HashLoginService("Hyrule"); + loginService.setUserStore(userStore); - HashUserRealm realm = new HashUserRealm(); - realm.setName("Hyrule"); - realm.put("Mr. Happy Pants", "xyzzy"); - realm.addUserToRole("Mr. Happy Pants", "user"); - - SecurityHandler securityHandler = new SecurityHandler(); - securityHandler.setUserRealm(realm); + ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler(); + securityHandler.setLoginService(loginService); securityHandler.setConstraintMappings(new ConstraintMapping[]{cm}); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { securityHandler, rhandler, servletHandler, diff --git a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java index 0771360c..7f09dfbe 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java +++ b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java @@ -38,12 +38,12 @@ import org.archive.io.arc.ARCReaderFactory; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; /** * Base class for 'self tests', integrations tests formatted as unit @@ -182,7 +182,7 @@ public abstract class SelfTestBase extends TmpDirTestCase { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index 3e66a064..005a5928 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -48,12 +48,12 @@ public class StatisticsSelfTest extends SelfTestBase { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); + assertEquals(7501, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(2133, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(5368, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); } @@ -66,17 +66,17 @@ public class StatisticsSelfTest extends SelfTestBase { sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(2942l, (long) sourceStats.get("novel")); + assertEquals(2133l, (long) sourceStats.get("novel")); assertEquals(3l, (long) sourceStats.get("novelCount")); - assertEquals(2942l, (long) sourceStats.get("warcNovelContentBytes")); + assertEquals(2133l, (long) sourceStats.get("warcNovelContentBytes")); assertEquals(3l, (long) sourceStats.get("warcNovelUrls")); sourceStats = stats.getSourceStats("http://localhost:7777/b.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); + assertEquals(5368l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); + assertEquals(5368l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); } diff --git a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java index 48925345..d70ddc0b 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java @@ -20,14 +20,14 @@ package org.archive.crawler.selftest; import org.archive.util.ArchiveUtils; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; /** * @author pjack @@ -54,17 +54,17 @@ public class UserAgentSelfTest extends SelfTestBase { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java index 7d3ce576..39ca0bb6 100644 --- a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java +++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java @@ -44,20 +44,20 @@ import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.NCSARequestLog; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.handler.RequestLogHandler; -import org.mortbay.jetty.security.Authenticator; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.FormAuthenticator; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.servlet.HashSessionManager; -import org.mortbay.jetty.servlet.SessionHandler; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.security.authentication.FormAuthenticator; +import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.handler.RequestLogHandler; +import org.eclipse.jetty.server.session.DefaultSessionCache; +import org.eclipse.jetty.server.session.NullSessionDataStore; +import org.eclipse.jetty.server.session.SessionCache; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; /* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but * the code is written, it's easier to run in eclipse, and no doubt tests @@ -211,12 +211,9 @@ public class FormAuthTest extends TestCase { public FormAuthTestHandler() { super(); } - + @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (target.endsWith("/set-cookie")) { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } @@ -247,16 +244,16 @@ public class FormAuthTest extends TestCase { constraintMapping.setConstraint(constraint); constraintMapping.setPathSpec("/auth/*"); - SecurityHandler authWrapper = new SecurityHandler(); + UserStore userStore = new UserStore(); + userStore.addUser(login, new Password(password), new String[]{role}); + + HashLoginService loginService = new HashLoginService(realm); + loginService.setUserStore(userStore); + + ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); - authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); - authWrapper.setUserRealm(new HashUserRealm(realm) { - { - put(login, password); - addUserToRole(login, role); - } - }); + authWrapper.setLoginService(loginService); return authWrapper; } @@ -265,7 +262,7 @@ public class FormAuthTest extends TestCase { // server for form auth Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7779); server.addConnector(sc); @@ -277,16 +274,17 @@ public class FormAuthTest extends TestCase { requestLogHandler.setRequestLog(requestLog); handlers.addHandler(requestLogHandler); - FormAuthenticator formAuthenticatrix = new FormAuthenticator(); - formAuthenticatrix.setLoginPage("/login.html"); - + FormAuthenticator formAuthenticatrix = new FormAuthenticator("/login.html", null, false); + SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix, FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN, FORM_AUTH_PASSWORD); authWrapper.setHandler(handlers); SessionHandler sessionHandler = new SessionHandler(); - sessionHandler.setSessionManager(new HashSessionManager()); + SessionCache cache = new DefaultSessionCache(sessionHandler); + cache.setSessionDataStore(new NullSessionDataStore()); + sessionHandler.setSessionCache(cache); sessionHandler.setHandler(authWrapper); server.setHandler(sessionHandler); diff --git a/engine/src/test/resources/log4j.xml b/engine/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/engine/src/test/resources/log4j.xml +++ b/engine/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/modules/pom.xml b/modules/pom.xml index 46ad49c7..1c7759b1 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -36,22 +36,14 @@ 1.6.3 - org.mortbay.jetty - jetty-util - 6.1.26 - test + org.eclipse.jetty + jetty-server + 9.4.19.v20190610 - org.mortbay.jetty - jetty-sslengine - 6.1.26 - test - - - org.mortbay.jetty - jetty - 6.1.26 - test + org.eclipse.jetty + jetty-security + 9.4.19.v20190610 org.littleshoot @@ -77,10 +69,13 @@ org.apache.maven.plugins maven-surefire-plugin - 2.9 + 2.22.2 **/TestAll.java + + **/*Tests.java diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 4ea94d22..257c8502 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -43,18 +43,17 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.SessionHandler; -import org.mortbay.log.Log; import com.google.common.io.Files; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.ssl.SslContextFactory; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { @@ -64,9 +63,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { } @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (request.getParameter("name") != null) { Cookie cookie = new javax.servlet.http.Cookie(request.getParameter("name"), request.getParameter("value")); @@ -114,7 +111,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { server.setHandler(new TestHandler()); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); @@ -133,11 +130,18 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - SslSocketConnector ssc = new SslSocketConnector(); + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); + sslContextFactory.setKeyStorePath(keystoreFile.getPath()); + + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + ServerConnector ssc = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); ssc.setHost("127.0.0.1"); ssc.setPort(7443); - ssc.setKeyPassword(KEYSTORE_PASSWORD); - ssc.setKeystore(keystoreFile.getPath()); server.addConnector(sc); server.addConnector(ssc); @@ -345,7 +349,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -379,7 +383,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=.example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=.example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=.example.com\r\n")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -413,7 +417,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=somethingelse.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=somethingelse.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=somethingelse.com\r\n")); // check fetch of original domain has no cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -434,7 +438,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=BAR.example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=bar.example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=bar.example.com\r\n")); // check fetch of original domain has no cookie curi = makeCrawlURI("http://foo.example.com:7777/"); @@ -463,7 +467,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n")); curi = makeCrawlURI("http://FOO.example.com:7777/"); fetcher().process(curi); diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java index a621a57d..35ff08d0 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java @@ -338,60 +338,7 @@ public class CookieStoreTest extends TmpDirTestCase { assertTrue(bdbCookieList.size() > 3000); assertCookieListsEquivalent(bdbCookieList, basicCookieStore().getCookies()); } - - public void testConcurrentLoad() throws IOException, InterruptedException { - bdbCookieStore().clear(); - basicCookieStore().clear(); - final Random rand = new Random(); - Runnable runnable = new Runnable() { - @Override - public void run() { - try { - while (!Thread.interrupted()) { - BasicClientCookie cookie = new BasicClientCookie(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - cookie.setDomain("d" + rand.nextInt(20) + ".example.com"); - bdbCookieStore().addCookie(cookie); - basicCookieStore().addCookie(cookie); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - - Thread[] threads = new Thread[200]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(runnable); - threads[i].setName("cookie-load-test-" + i); - threads[i].start(); - } - - Thread.sleep(1000); - - for (int i = 0; i < threads.length; i++) { - threads[i].interrupt(); - } - for (int i = 0; i < threads.length; i++) { - threads[i].join(); - } - - ArrayList bdbCookieArrayList = new ArrayList(bdbCookieStore().getCookies()); - Map domainCounts = new HashMap(); - for (Cookie cookie : bdbCookieArrayList) { - if (domainCounts.get(cookie.getDomain()) == null) { - domainCounts.put(cookie.getDomain(), 1); - } - else { - domainCounts.put(cookie.getDomain(), domainCounts.get(cookie.getDomain()) + 1); - } - } - - for (String domain: domainCounts.keySet()) { - assertTrue(domainCounts.get(domain) <= BdbCookieStore.MAX_COOKIES_FOR_DOMAIN + 25); - } - } - protected void assertCookieStoreCountEquals(BdbCookieStore bdb, int count) { assertEquals(bdb.getCookies().size(), count); } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 0b159abe..08b7c4a6 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -30,27 +30,21 @@ import javax.servlet.http.HttpServletResponse; import org.archive.modules.ProcessorTestBase; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.NCSARequestLog; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Response; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.handler.RequestLogHandler; -import org.mortbay.jetty.security.Authenticator; -import org.mortbay.jetty.security.BasicAuthenticator; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.DigestAuthenticator; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.SessionHandler; -import org.mortbay.log.Log; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.security.authentication.BasicAuthenticator; +import org.eclipse.jetty.security.authentication.DigestAuthenticator; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; +import org.eclipse.jetty.util.ssl.SslContextFactory; public class FetchHTTPTest extends ProcessorTestBase { @@ -125,12 +119,13 @@ public class FetchHTTPTest extends ProcessorTestBase { public TestHandler() { super(); } - + @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + + // echo the remote host back to the client so tests can reference it + response.setHeader("Client-Host", request.getRemoteHost()); + if (target.endsWith("/set-cookie")) { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } @@ -221,9 +216,7 @@ public class FetchHTTPTest extends ProcessorTestBase { } protected static Map httpServers; - protected static Request lastRequest = null; - protected static Response lastResponse = null; - + protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, final String role, String realm, final String login, final String password) { @@ -235,16 +228,15 @@ public class FetchHTTPTest extends ProcessorTestBase { constraintMapping.setConstraint(constraint); constraintMapping.setPathSpec("/auth/*"); - SecurityHandler authWrapper = new SecurityHandler(); + ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); - authWrapper.setUserRealm(new HashUserRealm(realm) { - { - put(login, password); - addUserToRole(login, role); - } - }); + UserStore userStore = new UserStore(); + userStore.addUser(login, new Password(password), new String[] {role}); + HashLoginService loginService = new HashLoginService(realm); + loginService.setUserStore(userStore); + authWrapper.setLoginService(loginService); return authWrapper; } @@ -259,24 +251,10 @@ public class FetchHTTPTest extends ProcessorTestBase { HashMap servers = new HashMap(); - HandlerCollection handlers = new HandlerCollection(); - handlers.addHandler(new TestHandler()); - RequestLogHandler requestLogHandler = new RequestLogHandler(); - NCSARequestLog requestLog = new NCSARequestLog() { - @Override - public void log(Request request, Response response) { - super.log(request, response); - lastRequest = request; - lastResponse = response; - } - }; - requestLogHandler.setRequestLog(requestLog); - handlers.addHandler(requestLogHandler); - // server for basic auth Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); @@ -284,6 +262,8 @@ public class FetchHTTPTest extends ProcessorTestBase { SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(), BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN, BASIC_AUTH_PASSWORD); + HandlerCollection handlers = new HandlerCollection(); + handlers.addHandler(new TestHandler()); authWrapper.setHandler(handlers); server.setHandler(authWrapper); @@ -299,16 +279,23 @@ public class FetchHTTPTest extends ProcessorTestBase { "-storepass", KEYSTORE_PASSWORD, "-keypass", KEYSTORE_PASSWORD, "-alias", "jetty", - "-genkey", + "-genkey", "-keyalg", "RSA", "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - - SslSocketConnector ssc = new SslSocketConnector(); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); + sslContextFactory.setKeyStorePath(keystoreFile.getPath()); + + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + ServerConnector ssc = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); ssc.setHost("127.0.0.1"); ssc.setPort(7443); - ssc.setKeyPassword(KEYSTORE_PASSWORD); - ssc.setKeystore(keystoreFile.getPath()); server.addConnector(ssc); @@ -317,7 +304,7 @@ public class FetchHTTPTest extends ProcessorTestBase { // server for digest auth server = new Server(); - sc = new SocketConnector(); + sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7778); server.addConnector(sc); @@ -325,7 +312,9 @@ public class FetchHTTPTest extends ProcessorTestBase { authWrapper = makeAuthWrapper(new DigestAuthenticator(), DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN, DIGEST_AUTH_PASSWORD); - authWrapper.setHandler(handlers); + HandlerCollection handlers2 = new HandlerCollection(); + handlers2.addHandler(new TestHandler()); + authWrapper.setHandler(handlers2); server.setHandler(authWrapper); server.start(); @@ -333,7 +322,7 @@ public class FetchHTTPTest extends ProcessorTestBase { return servers; } - + protected static void ensureHttpServers() throws Exception { if (httpServers == null) { httpServers = startHttpServers(); @@ -362,9 +351,4 @@ public class FetchHTTPTest extends ProcessorTestBase { } }; } - - public static Request getLastRequest() { - return lastRequest; - } - } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 43de8044..6ca6728c 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -157,7 +157,7 @@ public class FetchHTTPTests extends ProcessorTestBase { assertEquals(DEFAULT_PAYLOAD_STRING, curi.getRecorder().getContentReplayCharSequence().toString()); if (!exclusions.contains("httpBindAddress")) { - assertEquals("127.0.0.1", FetchHTTPTest.getLastRequest().getRemoteAddr()); + assertTrue(rawResponseString(curi).contains("Client-Host: 127.0.0.1\r\n")); } if (!exclusions.contains("nonFatalFailuresIsEmpty")) { @@ -262,7 +262,7 @@ public class FetchHTTPTests extends ProcessorTestBase { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); - assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); + assertEquals("basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); assertTrue(curi.getCredentials().contains(basicAuthCredential)); assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic")); @@ -406,8 +406,8 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); // the client bind address isn't recorded anywhere in heritrix as - // far as i can tell, so we get it this way... - assertEquals(addr, FetchHTTPTest.getLastRequest().getRemoteAddr()); + // far as i can tell, so we get the server to echo it back to us... + assertTrue(rawResponseString(curi).contains("Client-Host: " + addr + "\r\n")); runDefaultChecks(curi, "httpBindAddress"); } @@ -780,7 +780,9 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertTrue(httpRequestString(curi).startsWith("GET /99% HTTP/1.0\r\n")); - runDefaultChecks(curi, "requestLine"); + // jetty 9 rejects requests with paths like this with 400 Bad Request + // so we can't run these checks anymore + //runDefaultChecks(curi, "requestLine"); } public void testTwoQuestionMarks() throws Exception { diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index 049a0610..85948e4f 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -70,11 +70,11 @@ import org.archive.spring.ConfigPath; import org.archive.util.Base32; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.servlet.SessionHandler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; public class ContentDigestHistoryTest extends TmpDirTestCase { @@ -237,7 +237,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { fetcher.process(curi1); assertEquals(200, curi1.getFetchStatus()); - assertEquals(141, curi1.getContentSize()); assertEquals(expectedDigest, curi1.getContentDigestSchemeString()); assertFalse(curi1.hasContentDigestHistory()); @@ -260,7 +259,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { fetcher.process(curi2); assertEquals(200, curi1.getFetchStatus()); - assertEquals(141, curi1.getContentSize()); assertEquals(expectedDigest, curi1.getContentDigestSchemeString()); assertFalse(curi2.hasContentDigestHistory()); @@ -305,7 +303,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { assertTrue(recordIterator.hasNext()); record = recordIterator.next(); assertEquals(WARCRecordType.response.toString(), record.getHeader().getHeaderValue(HEADER_KEY_TYPE)); - assertEquals("141", record.getHeader().getHeaderValue(CONTENT_LENGTH)); assertEquals(expectedDigest, record.getHeader().getHeaderValue(HEADER_KEY_PAYLOAD_DIGEST)); assertEquals(curi1.getUURI().toString(), record.getHeader().getHeaderValue(HEADER_KEY_URI)); assertEquals(payloadRecordIdWithBrackets, record.getHeader().getHeaderValue(HEADER_KEY_ID)); @@ -374,10 +371,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { HandlerCollection handlers = new HandlerCollection(); handlers.addHandler(new SessionHandler(){ @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); @@ -388,7 +382,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { Server server = new Server(); server.setHandler(handlers); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); diff --git a/modules/src/test/resources/log4j.xml b/modules/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/modules/src/test/resources/log4j.xml +++ b/modules/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/pom.xml b/pom.xml index 033cd488..2cf5e564 100644 --- a/pom.xml +++ b/pom.xml @@ -373,15 +373,15 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html maven-compiler-plugin 3.3 - 1.7 - 1.7 + 1.8 + 1.8 UTF-8 org.apache.maven.plugins maven-surefire-plugin - 2.9 + 2.22.2 -Xmx1g