From 1ed207976c539dd282268d98db4307b2f195de23 Mon Sep 17 00:00:00 2001 From: nruest Date: Mon, 22 Apr 2019 09:17:23 -0400 Subject: [PATCH 01/23] Update README and add LICENSE.txt - Add badges to README - Update Markdown - Clean-up formatting - Add LICENSE.txt in root - Partially addresses #233 --- LICENSE | 11 +++++++ README.md | 85 +++++++++++++++++++------------------------------------ 2 files changed, 40 insertions(+), 56 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..90edcee4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,11 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 5f307433..c1a47653 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,37 @@ -Readme for Heritrix -==================== +# Heritrix +[![Build Status](https://travis-ci.org/internetarchive/heritrix3.svg?branch=master)](https://travis-ci.org/internetarchive/heritrix3) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix) +[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](http://builds.archive.org/javadoc/heritrix-3.2.0/) +[![LICENSE](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](./LICENSE) -1. Introduction -2. Crawl Operators! -3. Getting Started -4. Developer Documentation -5. Release History -6. License +## Introduction + +Heritrix is the Internet Archive's open-source, extensible, web-scale, archival-quality web crawler project. Heritrix (sometimes spelled heretrix, or misspelled or missaid as heratrix/heritix/heretix/heratix) is an archaic word for heiress (woman who inherits). Since our crawler seeks to collect and preserve the digital artifacts of our culture for the benefit of future researchers and generations, this name seemed apt. + +## Crawl Operators! + +Heritrix is designed to respect the [`robots.txt`](http://www.robotstxt.org/wc/robots.html) exclusion directives and [META robots tags](http://www.robotstxt.org/wc/exclusion.html#meta). Please consider the +load your crawl will place on seed sites and set politeness policies accordingly. Also, always identify your crawl with contact information in the `User-Agent` so sites that may be adversely affected by your crawl can contact you or adapt their server behavior accordingly. + +## Getting Started + +- [User Manual](https://github.com/internetarchive/heritrix3/wiki) + +## Developer Documentation + +- [Developer Manual](http://crawler.archive.org/articles/developer_manual/index.html) +- [REST API documentation](https://heritrix.readthedocs.io/en/latest/api.html) +- [JavaDoc](http://builds.archive.org/javadoc/heritrix-3.2.0/) (n.b. Javadoc currently out of date) -## 1. Introduction +## Latest Releases -Heritrix is the Internet Archive's open-source, extensible, web-scale, -archival-quality web crawler project. Heritrix (sometimes spelled heretrix, or -misspelled or missaid as heratrix/heritix/heretix/heratix) is an archaic word -for heiress (woman who inherits). Since our crawler seeks to collect and -preserve the digital artifacts of our culture for the benefit of future -researchers and generations, this name seemed apt. +Information about releases can be found [here](https://github.com/internetarchive/heritrix3/wiki#latest-releases). +## License -## 2. Crawl Operators! +Heritrix is free software; you can redistribute it and/or modify it under the terms of the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) -Heritrix is designed to respect the robots.txt - exclusion directives and META robots -tags . Please consider the -load your crawl will place on seed sites and set politeness policies -accordingly. Also, always identify your crawl with contact information in the -User-Agent so sites that may be adversely affected by your crawl can contact -you or adapt their server behavior accordingly. - - -## 3. Getting Started - -See the User Manual, available from - - -## 4. Developer Documentation - -See . -For REST API documentation, see -and for JavaDoc see (n.b. Javadoc currently out of date). - - -## 5. Latest Releases - -Information about releases can be found at - - -## 6. License - -Heritrix is free software; you can redistribute it and/or modify it -under the terms of the Apache License, Version 2.0: - - http://www.apache.org/licenses/LICENSE-2.0 - -Some individual source code files are subject to or offered under other -licenses. See the included LICENSE.txt file for more information. - -Heritrix is distributed with the libraries it depends upon. The -libraries can be found under the 'lib' directory, and are used under -the terms of their respective licenses, which are included alongside -the libraries in the 'lib' directory. +Some individual source code files are subject to or offered under other licenses. See the included [`LICENSE.txt`](./LICENSE) file for more information. +Heritrix is distributed with the libraries it depends upon. The libraries can be found under the `lib` directory in the release distribution, and are used under the terms of their respective licenses, which are included alongside the libraries in the `lib` directory. From 74a0222e5686a2e28cfe517038e211fc62cc0dc0 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 2 Aug 2019 15:47:02 +0900 Subject: [PATCH 02/23] Travis fixes - use trusty dist image for oraclejdk8 and openjdk7 as they're [unavailable] in the new xenial image - jdk7: skip building contrib rather than allowing failures - contrib is now in the parent pom so remove its separate build command - start testing against openjdk11 but allow failures for now [unavailable]: https://travis-ci.community/t/install-of-oracle-jdk-8-failing/3038/9 --- .travis.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 43c4eec6..20978aa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,19 @@ + sudo: false language: java - -jdk: - - oraclejdk8 - - openjdk7 - - openjdk8 - matrix: - allow_failures: + include: + - jdk: oraclejdk8 + dist: trusty - jdk: openjdk7 - + dist: trusty + env: PROJECTS='--projects commons,modules,engine' + - jdk: openjdk8 + - jdk: openjdk11 + allow_failures: + - jdk: openjdk11 + before_install: - "export JAVA_OPTS=-Xmx1500m" - "echo JAVA_OPTS=$JAVA_OPTS" @@ -18,14 +21,15 @@ before_install: - "echo MAVEN_OPTS=$MAVEN_OPTS" - "export _JAVA_OPTIONS=-Xmx1500m" - "echo _JAVA_OPTIONS=$_JAVA_OPTIONS" + +install: mvn dependency:resolve -B -V $PROJECTS cache: directories: - $HOME/.m2 script: - - travis_wait 30 mvn install - - cd contrib && mvn install + - travis_wait 30 mvn install $PROJECTS after_failure: - cat */target/surefire-reports/*.txt From 412885fd7d03fbb14b04d3e58856c4b27cc04853 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 2 Aug 2019 16:43:20 +0900 Subject: [PATCH 03/23] JDK11 support: upgrade maven-surefire-plugin to 2.22.2 Under JDK11 our old version of it throws ClassNotFoundExceptions when tests load some builtin classes like javax.transaction.xa.Xid. Fixes #266 --- commons/pom.xml | 2 +- dist/pom.xml | 2 +- engine/pom.xml | 2 +- modules/pom.xml | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index 3a9da7dc..3a121b02 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -221,7 +221,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.9 + 2.22.2 + + javax.xml.bind + jaxb-api + 2.3.1 + + + org.glassfish.jaxb + jaxb-runtime + 2.3.1 + runtime + org.littleshoot littleproxy From bf819e19a5ca4adcf4c2c1e3b4b8a56b2bba87e9 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 2 Aug 2019 13:44:49 +0900 Subject: [PATCH 05/23] JDK11 support: exclude tools.jar from hbase-client dependency tools.jar is no longer included in the JDK as of JDK 11 (per JEP 220) so exclude it as a dependency. According to [HBASE-13963] it is only required when compiling hbase itself and was unintentionally leaked as a transitive dependency. Fixes #265 [HBASE-13963]: https://issues.apache.org/jira/browse/HBASE-13963 --- contrib/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/pom.xml b/contrib/pom.xml index 5a1e1ba6..f6afde53 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -26,6 +26,13 @@ junit junit + + + jdk.tools + jdk.tools + From a5c2b4620876457f1e54235ff787064a875c7493 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 2 Aug 2019 17:10:13 +0900 Subject: [PATCH 06/23] JDK11 support: remove unused class ObjectIdentityBdbCache and tests In 2011 ObjectIdentityBdbCache was replaced by ObjectIdentityBdbManualCache in order to "remove dependence on GC/finalization/PhantomReference magic". It hasn't been used since. As of JDK11 the ObjectIdentityBdbCache unit tests are failing so let's remove it. --- .../archive/util/ObjectIdentityBdbCache.java | 564 ------------------ .../util/ObjectIdentityBdbCacheTest.java | 199 ------ 2 files changed, 763 deletions(-) delete mode 100644 commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java delete mode 100644 commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java deleted file mode 100644 index fd707c53..00000000 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java +++ /dev/null @@ -1,564 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.Closeable; -import java.io.Serializable; -import java.lang.ref.PhantomReference; -import java.lang.ref.Reference; -import java.lang.ref.ReferenceQueue; -import java.lang.ref.SoftReference; -import java.lang.reflect.Field; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.archive.bdb.KryoBinding; - -import com.sleepycat.bind.EntryBinding; -import com.sleepycat.bind.serial.StoredClassCatalog; -import com.sleepycat.bind.tuple.TupleBinding; -import com.sleepycat.collections.StoredSortedMap; -import com.sleepycat.je.Database; -import com.sleepycat.je.DatabaseConfig; -import com.sleepycat.je.DatabaseException; -import com.sleepycat.je.Environment; - -/** - * A BDB JE backed object cache. - * - * Soft references to previously-instantiated objects are held so that - * unless/until an object is garbage collected, subsequent get()s will - * return the exact same object. (If all outside references are lost, - * when the soft reference is broken, the object state -- still - * accessible to this class via reflective access to a phantom - * referent --is flushed to disk. The next get() will reconsitute a new - * object, from the disk state.) - *

- * The backing disk is only guaranteed to be up-to-date after a flush - * of all in-memory values to disk, as can be forced by sync(). - *

- * To ensure that changes/mutations to values in this map are coherent and - * consistent at the application level, it is assumed that the application - * level only mutates values that are in this map and does not retain references - * to values longer than necessary. This allows mappings to be persisted - * during GC without explicit transactions or write operations. - *

- * Based on the earlier CachedBdbMap. - *

- * - * @author John Erik Halse - * @author stack - * @author gojomo - * @author paul baclace (conversion to ConcurrentMap) - * - */ -public class ObjectIdentityBdbCache -implements ObjectIdentityCache, Closeable, Serializable { - private static final long serialVersionUID = 1L; - private static final Logger logger = - Logger.getLogger(ObjectIdentityBdbCache.class.getName()); - - /** The BDB JE database used for this instance. */ - protected transient Database db; - - /** in-memory map of new/recent/still-referenced-elsewhere instances */ - protected transient ConcurrentHashMap> memMap; - protected transient ReferenceQueue refQueue; - - /** The Collection view of the BDB JE database used for this instance. */ - protected transient StoredSortedMap diskMap; - - protected AtomicLong count; - - // - // USAGE STATS - // - /** Count of times we got an object from in-memory cache */ - private AtomicLong cacheHit = new AtomicLong(0); - /** Count of times the {@link ObjectIdentityBdbCache#get} method was called. */ - private AtomicLong countOfGets = new AtomicLong(0); - /** Count of every time disk-based map provided non-null object */ - private AtomicLong diskHit = new AtomicLong(0); - /** Count of times Supplier was used for new object */ - private AtomicLong supplierUsed = new AtomicLong(0); - /** count of expunge put() to BDB (implies disk) */ - private AtomicLong expungeStatsDiskPut = new AtomicLong(0); - /** count of {@link #sync()} use */ - transient private AtomicLong useStatsSyncUsed = new AtomicLong(0); - - /** Reference to the Reference#referent Field. */ - protected static Field referentField; - static { - // We need access to the referent field in the PhantomReference. - // For more on this trick, see - // - // http://www.javaspecialists.co.za/archive/Issue098.html and for - // discussion: - // http://www.theserverside.com/tss?service=direct/0/NewsThread/threadViewer.markNoisy.link&sp=l29865&sp=l146901 - try { - referentField = Reference.class.getDeclaredField("referent"); - referentField.setAccessible(true); - } catch (SecurityException e) { - throw new RuntimeException(e); - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - } - - /** - * Constructor. You must call - * {@link #initialize(Environment, String, Class, StoredClassCatalog)} - * to finish construction. Construction is two-stepped to support - * reconnecting a deserialized CachedBdbMap with its backing bdbje - * database. - */ - public ObjectIdentityBdbCache() { - super(); - } - - /** - * Call this method when you have an instance when you used the - * default constructor or when you have a deserialized instance that you - * want to reconnect with an extant bdbje environment. - * @param env - * @param dbName - * @param valueClass - * @param classCatalog - * @throws DatabaseException - */ - public void initialize(final Environment env, String dbName, - final Class valueClass, final StoredClassCatalog classCatalog) - throws DatabaseException { - // TODO: initial capacity should be related to number of seeds, max depth, max docs - this.memMap = new ConcurrentHashMap>( - 8192, // initial capacity - 0.9f, // acceptable load factor - 64 // est. number of concurrent threads - ); - this.refQueue = new ReferenceQueue(); - canary = new SoftReference(new LowMemoryCanary()); - - this.db = openDatabase(env, dbName); - this.diskMap = createDiskMap(this.db, classCatalog, valueClass); - this.count = new AtomicLong(diskMap.size()); - } - - @SuppressWarnings("unchecked") - protected StoredSortedMap createDiskMap(Database database, - StoredClassCatalog classCatalog, Class valueClass) { - EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(String.class); - EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass); - if(valueBinding == null) { - valueBinding = - new KryoBinding(valueClass); -// new SerialBinding(classCatalog, valueClass); -// new BenchmarkingBinding(new EntryBinding[] { -// new KryoBinding(valueClass), -// new RecyclingSerialBinding(classCatalog, valueClass), -// }, valueClass); - } - return new StoredSortedMap(database, keyBinding, valueBinding, true); - } - - protected Database openDatabase(final Environment environment, - final String dbName) throws DatabaseException { - DatabaseConfig dbConfig = new DatabaseConfig(); - dbConfig.setTransactional(false); - dbConfig.setAllowCreate(true); - dbConfig.setDeferredWrite(true); - return environment.openDatabase(null, dbName, dbConfig); - } - - /* (non-Javadoc) - * @see org.archive.util.ObjectIdentityCache#close() - */ - public synchronized void close() { - // Close out my bdb db. - if (this.db != null) { - try { - sync(); - this.db.sync(); - this.db.close(); - } catch (DatabaseException e) { - logger.log(Level.WARNING,"problem closing ObjectIdentityBdbCache",e); - } finally { - this.db = null; - } - } - } - - protected void finalize() throws Throwable { - close(); - super.finalize(); - } - - /* (non-Javadoc) - * @see org.archive.util.ObjectIdentityCache#get(java.lang.String) - */ - public V get(final String key) { - return getOrUse(key,null); - } - - /* (non-Javadoc) - * @see org.archive.util.ObjectIdentityCache#get(java.lang.String, org.archive.util.ObjectIdentityBdbCache) - */ - public V getOrUse(final String key, Supplier supplierOrNull) { - countOfGets.incrementAndGet(); - - if (countOfGets.get() % 10000 == 0) { - logCacheSummary(); - } - - // check mem cache - SoftEntry entry = memMap.get(key); - if(entry != null) { - V val = entry.get(); - if(val != null) { - // the concurrent garden path: in mem, valid - cacheHit.incrementAndGet(); - val.setIdentityCache(this); - return val; - } - } - - // everything in other difficult cases happens inside this block - synchronized(this) { - // recheck mem cache -- if another thread beat us into sync - // block and already filled the key - entry = memMap.get(key); - if(entry != null) { - V val = entry.get(); - if(val != null) { - cacheHit.incrementAndGet(); - val.setIdentityCache(this); - return val; - } - } - // persist to disk all ref-enqueued stale (soft-ref-cleared) entries now - pageOutStaleEntries(); - // and catch if this exact entry not yet ref-enqueued - if(memMap.get(key)!=null) { - pageOutStaleEntry(entry); - if(memMap.get(key)!=null) { - logger.log(Level.SEVERE,"nulled key "+key+" not paged-out", new Exception()); - } - } - - // check disk - V valDisk = (V) diskMap.get(key); - if(valDisk==null) { - // never yet created, consider creating - if(supplierOrNull==null) { - return null; - } - // create using provided Supplier - valDisk = supplierOrNull.get(); - supplierUsed.incrementAndGet(); - // putting initial value directly into diskMap - // (rather than just the memMap until page-out) - // ensures diskMap.keySet() provides complete view - V prevVal = diskMap.putIfAbsent(key, valDisk); - count.incrementAndGet(); - if(prevVal!=null) { - // ERROR: diskMap modification since previous - // diskMap.get() should be impossible - logger.log(Level.SEVERE,"diskMap modified outside synchronized block?"); - } - } else { - diskHit.incrementAndGet(); - } - - // keep new val in memMap - SoftEntry newEntry = new SoftEntry(key, valDisk, refQueue); - SoftEntry prevVal = memMap.putIfAbsent(key, newEntry); - if(prevVal != null) { - // ERROR: memMap modification since previous - // memMap.get() should be impossible - logger.log(Level.SEVERE,"memMap modified outside synchronized block?", new Exception()); - } - valDisk.setIdentityCache(this); - return valDisk; - } - } - - /* (non-Javadoc) - * @see org.archive.util.ObjectIdentityCache#keySet() - */ - public Set keySet() { - return diskMap.keySet(); - } - - /** - * Summary to log, if at FINE level - */ - private void logCacheSummary() { - if (logger.isLoggable((Level.FINE))) { - logger.fine(composeCacheSummary()); - } - } - - protected String composeCacheSummary() { - long totalHits = cacheHit.get() + diskHit.get(); - if (totalHits < 1) { - return ""; - } - long cacheHitPercent - = (cacheHit.get() * 100) / totalHits; - StringBuilder sb = new StringBuilder(120); - sb.append("DB name:") - .append(getDatabaseName()) - .append(", ") - .append(" hit%: ") - .append(cacheHitPercent) - .append("%, gets=") - .append(countOfGets.get()) - .append(" memHits=") - .append(cacheHit.get()) - .append(" diskHits=") - .append(diskHit.get()) - .append(" supplieds=") - .append(supplierUsed.get()) - .append(" expungePuts=") - .append(expungeStatsDiskPut.get()) - .append(" syncs=") - .append(useStatsSyncUsed.get()); - return sb.toString(); - } - - /* (non-Javadoc) - * @see org.archive.util.ObjectIdentityCache#size() - */ - public int size() { - if(db==null) { - return 0; - } - return (int) count.get(); - } - - protected String getDatabaseName() { - String name = "DbName-Lookup-Failed"; - try { - if (this.db != null) { - name = this.db.getDatabaseName(); - } - } catch (DatabaseException e) { - // Ignore. - } - return name; - } - - /** - * Sync all in-memory map entries to backing disk store. - */ - public synchronized void sync() { - String dbName = null; - // Sync. memory and disk. - useStatsSyncUsed.incrementAndGet(); - long startTime = 0; - if (logger.isLoggable(Level.FINE)) { - dbName = getDatabaseName(); - startTime = System.currentTimeMillis(); - logger.fine(dbName + " start sizes: disk " + this.diskMap.size() + - ", mem " + this.memMap.size()); - } - - for (String key : this.memMap.keySet()) { - SoftEntry entry = memMap.get(key); - if (entry != null) { - // Get & hold so not cleared pre-return. - V value = entry.get(); - if (value != null) { - expungeStatsDiskPut.incrementAndGet(); - this.diskMap.put(key, value); // unchecked cast - } - } - } - pageOutStaleEntries(); - - // force sync of deferred-writes - try { - this.db.sync(); - } catch (DatabaseException e) { - throw new RuntimeException(e); - } - - if (logger.isLoggable(Level.FINE)) { - logger.fine(dbName + " sync took " + - (System.currentTimeMillis() - startTime) + "ms. " + - "Finish sizes: disk " + - this.diskMap.size() + ", mem " + this.memMap.size()); - } - } - - @Override - public void dirtyKey(String key) { - // do nothing, because our weak/phantom trickery is supposed to - // ensure sync-to-persistence if/when dereferenced and collected - } - - /** An incremental, poll-based expunger. - * - * Package-protected for unit-test visibility. - */ - @SuppressWarnings("unchecked") - protected synchronized void pageOutStaleEntries() { - int c = 0; - long startTime = System.currentTimeMillis(); - for(SoftEntry entry; (entry = (SoftEntry)refQueue.poll()) != null;) { - pageOutStaleEntry(entry); - c++; - } - if (c > 0 && logger.isLoggable(Level.FINER)) { - long endTime = System.currentTimeMillis(); - try { - logger.finer("DB: " + db.getDatabaseName() + ", Expunged: " - + c + ", Diskmap size: " + diskMap.size() - + ", Cache size: " + memMap.size() - + ", in "+(endTime-startTime)+"ms"); - } catch (DatabaseException e) { - logger.log(Level.FINER,"exception while logging",e); - } - } - } - - /** - * Expunge an entry from memMap while updating diskMap. - * - * @param entry a SoftEntry obtained from refQueuePoll() - */ - synchronized private void pageOutStaleEntry(SoftEntry entry) { - PhantomEntry phantom = entry.phantom; - - // Still in memMap? if not, was paged-out by earlier direct access - // before placed into reference-queue; just return - if (memMap.get(phantom.key) != entry) { // NOTE: intentional identity compare - return; - } - - // recover hidden value - V phantomValue = phantom.doctoredGet(); - - // Expected value present? (should be; only clear is at end of - // this method, after entry removal from memMap) - if(phantomValue == null) { - logger.log(Level.WARNING,"unexpected null phantomValue", new Exception()); - return; // nothing to do - } - - // given instance entry still in memMap; - // we have the key and phantom Value, - // the diskMap can be updated. - diskMap.put(phantom.key, phantomValue); // unchecked cast - expungeStatsDiskPut.incrementAndGet(); - - // remove memMap entry - boolean removed = memMap.remove(phantom.key, entry); - if(!removed) { - logger.log(Level.WARNING,"expunge memMap.remove() ineffective",new Exception()); - } - phantom.clear(); // truly allows GC of unreferenced V object - } - - private static class PhantomEntry extends PhantomReference { - protected final String key; - - public PhantomEntry(String key, V referent) { - super(referent, null); - this.key = key; - } - - /** - * @return Return the referent. The contract for {@link #get()} - * always returns a null referent. We've cheated and doctored - * PhantomReference to return the actual referent value. See notes - * at {@link #referentField}; - */ - @SuppressWarnings("unchecked") - final public V doctoredGet() { - try { - // Here we use the referentField saved off on static - // initialization of this class to get at this References' - // private referent field. - return (V) referentField.get(this); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - } - - /** - * SoftReference cache entry. - * - * A PhantomReference is used to hold the key and value as a last - * chance before GC hook that can effect the update of diskMap. - *

- * Entries are not recycled. - */ - private static class SoftEntry extends SoftReference { - PhantomEntry phantom; - - public SoftEntry(String key, V referent, ReferenceQueue q) { - super(referent, q); - this.phantom = new PhantomEntry(key, referent); - } - - public V get() { - // ensure visibility - synchronized (this) { - return super.get(); - } - } - - public String toString() { - if (phantom != null) { - return "SoftEntry(key=" + phantom.key + ")"; - } else { - return "SoftEntry()"; - } - } - } - - // - // Crude, probably unreliable/fragile but harmless mechanism to - // trigger expunge of cleared SoftReferences in low-memory - // conditions even without any of the other get/put triggers. - // - - protected transient SoftReference canary; - protected class LowMemoryCanary { - /** When collected/finalized -- as should be expected in - * low-memory conditions -- trigger an expunge and a - * new 'canary' insertion. */ - public void finalize() { - ObjectIdentityBdbCache.this.pageOutStaleEntries(); -// System.err.println("CANARY KILLED - "+ObjectIdentityBdbCache.this); - // only install new canary if map still 'open' with db reference - if(ObjectIdentityBdbCache.this.db !=null) { - ObjectIdentityBdbCache.this.canary = - new SoftReference(new LowMemoryCanary()); - } else { - ObjectIdentityBdbCache.this.canary = null; - } - } - } -} diff --git a/commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java b/commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java deleted file mode 100644 index 1eb6e494..00000000 --- a/commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util; - -import java.io.File; -import java.util.HashMap; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.math.RandomUtils; -import org.archive.util.bdbje.EnhancedEnvironment; - -/** - * @author stack - * @author gojomo - * @version $Date: 2009-08-03 23:50:43 -0700 (Mon, 03 Aug 2009) $, $Revision: 6434 $ - */ -public class ObjectIdentityBdbCacheTest extends TmpDirTestCase { - EnhancedEnvironment env; - private ObjectIdentityBdbCache>> cache; - - protected void setUp() throws Exception { - super.setUp(); - File envDir = new File(getTmpDir(),"ObjectIdentityBdbCacheTest"); - org.archive.util.FileUtils.ensureWriteableDirectory(envDir); - FileUtils.deleteDirectory(envDir); - org.archive.util.FileUtils.ensureWriteableDirectory(envDir); - env = EnhancedEnvironment.getTestEnvironment(envDir); - this.cache = new ObjectIdentityBdbCache>>(); - this.cache.initialize(env,"setUpCache",IdentityCacheableWrapper.class, env.getClassCatalog()); - } - - protected void tearDown() throws Exception { - this.cache.close(); - File envDir = env.getHome(); - env.close(); - FileUtils.deleteDirectory(envDir); - super.tearDown(); - } - - @SuppressWarnings("unchecked") - public void testReadConsistencyUnderLoad() throws Exception { - final ObjectIdentityBdbCache> cbdbmap = - new ObjectIdentityBdbCache(); - cbdbmap.initialize(env, - "consistencyCache", - IdentityCacheableWrapper.class, - env.getClassCatalog()); - try { - final AtomicInteger level = new AtomicInteger(0); - final int keyCount = 128 * 1024; // 128K keys - final int maxLevel = 64; - // initial fill - for(int i=0; i < keyCount; i++) { - final String key = ""+i; - cbdbmap.getOrUse( - key, - new Supplier>( - new IdentityCacheableWrapper( - key, new AtomicInteger(level.get())))); - } - // backward checking that all values always at level or higher - new Thread() { - public void run() { - untilmax: while(true) { - for(int j=keyCount-1; j >= 0; j--) { - int targetValue = level.get(); - if(targetValue>=maxLevel) { - break untilmax; - } - assertTrue("stale value revseq key "+j,cbdbmap.get(""+j).get().get()>=targetValue); - Thread.yield(); - } - } - } - };//.start(); - // random checking that all values always at level or higher - new Thread() { - public void run() { - untilmax: while(true) { - int j = RandomUtils.nextInt(keyCount); - int targetValue = level.get(); - if(targetValue>=maxLevel) { - break untilmax; - } - assertTrue("stale value random key "+j, - cbdbmap.get(""+j).get().get()>=targetValue); - Thread.yield(); - } - } - };//.start(); - // increment all keys - for(; level.get() < maxLevel; level.incrementAndGet()) { - for(int k = 0; k < keyCount; k++) { - int foundValue = cbdbmap.get(""+k).get().getAndIncrement(); - assertEquals("stale value preinc key "+k, level.get(), foundValue); - } - if(level.get() % 10 == 0) { - System.out.println("level to "+level.get()); - if(level.get()>0) { - TestUtils.forceScarceMemory(); - } - System.out.println("OIBCT:"+cbdbmap.composeCacheSummary()); - } - Thread.yield(); - } - } finally { - System.err.println("OIBCT:"+cbdbmap.composeCacheSummary()); - cbdbmap.close(); - } - // SUCCESS - } - - public void testBackingDbGetsUpdated() { - // Set up values. - final String value = "value"; - final String key = "key"; - final int upperbound = 3; - // First put in empty hashmap. - for (int i = 0; i < upperbound; i++) { - String innerKey = key + Integer.toString(i); - this.cache.getOrUse( - innerKey, - new Supplier>>( - new IdentityCacheableWrapper>( - innerKey, new HashMap()))); - } - // Now add value to hash map. - for (int i = 0; i < upperbound; i++) { - HashMap m = this.cache.get(key + Integer.toString(i)).get(); - m.put(key, value); - } - this.cache.sync(); - for (int i = 0; i < upperbound; i++) { - HashMap m = this.cache.get(key + Integer.toString(i)).get(); - String v = m.get(key); - assertNotNull("value should not be null",v); - assertEquals("value incorrect", value, v); - } - } - - /** - * Test that in scarce memory conditions, the memory map is - * expunged of otherwise unreferenced entries as expected. - * - * NOTE: this test may be especially fragile with regard to - * GC/timing issues; relies on timely finalization, which is - * never guaranteed by JVM/GC. For example, it is so sensitive - * to CPU speed that a Thread.sleep(1000) succeeds when my - * laptop is plugged in, but fails when it is on battery! - * - * @throws InterruptedException - */ - public void testMemMapCleared() throws InterruptedException { - TestUtils.forceScarceMemory(); - System.gc(); // minimize effects of earlier test heap use - assertEquals(0, cache.memMap.size()); - assertEquals(0, cache.diskMap.size()); - for(int i=0; i < 10000; i++) { - String key = ""+i; - cache.getOrUse( - key, - new Supplier>>( - new IdentityCacheableWrapper>( - key, new HashMap()))); - } - assertEquals(cache.memMap.size(), 10000); - assertEquals(cache.size(), 10000); - TestUtils.forceScarceMemory(); - Thread.sleep(3000); - // The 'canary' trick may make this explicit page-out, or - // a page-out riggered by a get() or put...(), unnecessary -- - // but we include anyway. - cache.pageOutStaleEntries(); - System.out.println(cache.size()+","+cache.memMap.size()); - assertEquals("memMap not cleared", 0, cache.memMap.size()); - } - - - public static void main(String [] args) { - junit.textui.TestRunner.run(ObjectIdentityBdbCacheTest.class); - } -} From d3d19e224895df802653aff56cf48e4cd59ce4a4 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 2 Aug 2019 17:57:09 +0900 Subject: [PATCH 07/23] Prevent newer surefire from invoking FetchHTTPTests directly It seems older versions only matched *Test.java but the new version is now incorrectly including it. --- modules/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/pom.xml b/modules/pom.xml index 6ea632ad..0115535c 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -80,6 +80,9 @@ **/TestAll.java + + **/*Tests.java From 9d1341c731f1cb747128ad7534ce553b0736bee7 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 5 Aug 2019 12:57:31 +0900 Subject: [PATCH 08/23] JDK11 support: upgrade jetty to 9.4.19 in modules tests Fixes #268 ssl handshake_failure. Support for jdk11 was added in jetty 9.4.14 but we may as well bump to the latest stable version. In Jetty 9 it appears the request is logged after the response is sent. Thus it was racing with the assertions that check the client IP. So to fix this rather stashing the 'lastRequest' we just make the server echo the client's IP in a response header. Some other minor tweaks were needed due to changes in Jetty behaviour: - We stop checking the length of the raw response. It doesn't tell us anything and easily varies. - Jetty now generates Set-Cookie with a space after the ; - Jetty now lowercases the word "basic" in WWW-Authenticate header - testLaxUrlEncoding(): Jetty now rejects bad paths with a 400 error so we disable the response checks. The actual request line is still checked which is the important thing. Note: This patch does not affect the version of jetty used by the Heritrix admin console. That will be tackled separately. --- modules/pom.xml | 20 +--- .../CookieFetchHTTPIntegrationTest.java | 40 ++++--- .../modules/fetcher/FetchHTTPTest.java | 104 ++++++++---------- .../modules/fetcher/FetchHTTPTests.java | 12 +- .../recrawl/ContentDigestHistoryTest.java | 20 ++-- 5 files changed, 86 insertions(+), 110 deletions(-) diff --git a/modules/pom.xml b/modules/pom.xml index d38cd346..a9d777e8 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -35,23 +35,15 @@ 1.6.3 - org.mortbay.jetty - jetty-util - 6.1.26 - test + org.eclipse.jetty + jetty-server + 9.4.19.v20190610 - org.mortbay.jetty - jetty-sslengine - 6.1.26 - test + org.eclipse.jetty + jetty-security + 9.4.19.v20190610 - - org.mortbay.jetty - jetty - 6.1.26 - test - org.littleshoot littleproxy diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 4ea94d22..257c8502 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -43,18 +43,17 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.SessionHandler; -import org.mortbay.log.Log; import com.google.common.io.Files; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.ssl.SslContextFactory; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { @@ -64,9 +63,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { } @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (request.getParameter("name") != null) { Cookie cookie = new javax.servlet.http.Cookie(request.getParameter("name"), request.getParameter("value")); @@ -114,7 +111,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { server.setHandler(new TestHandler()); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); @@ -133,11 +130,18 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - SslSocketConnector ssc = new SslSocketConnector(); + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); + sslContextFactory.setKeyStorePath(keystoreFile.getPath()); + + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + ServerConnector ssc = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); ssc.setHost("127.0.0.1"); ssc.setPort(7443); - ssc.setKeyPassword(KEYSTORE_PASSWORD); - ssc.setKeystore(keystoreFile.getPath()); server.addConnector(sc); server.addConnector(ssc); @@ -345,7 +349,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -379,7 +383,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=.example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=.example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=.example.com\r\n")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -413,7 +417,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=somethingelse.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=somethingelse.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=somethingelse.com\r\n")); // check fetch of original domain has no cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -434,7 +438,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=BAR.example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=bar.example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=bar.example.com\r\n")); // check fetch of original domain has no cookie curi = makeCrawlURI("http://foo.example.com:7777/"); @@ -463,7 +467,7 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n")); curi = makeCrawlURI("http://FOO.example.com:7777/"); fetcher().process(curi); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 0b159abe..08b7c4a6 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -30,27 +30,21 @@ import javax.servlet.http.HttpServletResponse; import org.archive.modules.ProcessorTestBase; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.NCSARequestLog; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Response; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.handler.RequestLogHandler; -import org.mortbay.jetty.security.Authenticator; -import org.mortbay.jetty.security.BasicAuthenticator; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.DigestAuthenticator; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.SessionHandler; -import org.mortbay.log.Log; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.security.authentication.BasicAuthenticator; +import org.eclipse.jetty.security.authentication.DigestAuthenticator; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; +import org.eclipse.jetty.util.ssl.SslContextFactory; public class FetchHTTPTest extends ProcessorTestBase { @@ -125,12 +119,13 @@ public class FetchHTTPTest extends ProcessorTestBase { public TestHandler() { super(); } - + @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + + // echo the remote host back to the client so tests can reference it + response.setHeader("Client-Host", request.getRemoteHost()); + if (target.endsWith("/set-cookie")) { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } @@ -221,9 +216,7 @@ public class FetchHTTPTest extends ProcessorTestBase { } protected static Map httpServers; - protected static Request lastRequest = null; - protected static Response lastResponse = null; - + protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, final String role, String realm, final String login, final String password) { @@ -235,16 +228,15 @@ public class FetchHTTPTest extends ProcessorTestBase { constraintMapping.setConstraint(constraint); constraintMapping.setPathSpec("/auth/*"); - SecurityHandler authWrapper = new SecurityHandler(); + ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); - authWrapper.setUserRealm(new HashUserRealm(realm) { - { - put(login, password); - addUserToRole(login, role); - } - }); + UserStore userStore = new UserStore(); + userStore.addUser(login, new Password(password), new String[] {role}); + HashLoginService loginService = new HashLoginService(realm); + loginService.setUserStore(userStore); + authWrapper.setLoginService(loginService); return authWrapper; } @@ -259,24 +251,10 @@ public class FetchHTTPTest extends ProcessorTestBase { HashMap servers = new HashMap(); - HandlerCollection handlers = new HandlerCollection(); - handlers.addHandler(new TestHandler()); - RequestLogHandler requestLogHandler = new RequestLogHandler(); - NCSARequestLog requestLog = new NCSARequestLog() { - @Override - public void log(Request request, Response response) { - super.log(request, response); - lastRequest = request; - lastResponse = response; - } - }; - requestLogHandler.setRequestLog(requestLog); - handlers.addHandler(requestLogHandler); - // server for basic auth Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); @@ -284,6 +262,8 @@ public class FetchHTTPTest extends ProcessorTestBase { SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(), BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN, BASIC_AUTH_PASSWORD); + HandlerCollection handlers = new HandlerCollection(); + handlers.addHandler(new TestHandler()); authWrapper.setHandler(handlers); server.setHandler(authWrapper); @@ -299,16 +279,23 @@ public class FetchHTTPTest extends ProcessorTestBase { "-storepass", KEYSTORE_PASSWORD, "-keypass", KEYSTORE_PASSWORD, "-alias", "jetty", - "-genkey", + "-genkey", "-keyalg", "RSA", "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - - SslSocketConnector ssc = new SslSocketConnector(); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); + sslContextFactory.setKeyStorePath(keystoreFile.getPath()); + + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + ServerConnector ssc = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); ssc.setHost("127.0.0.1"); ssc.setPort(7443); - ssc.setKeyPassword(KEYSTORE_PASSWORD); - ssc.setKeystore(keystoreFile.getPath()); server.addConnector(ssc); @@ -317,7 +304,7 @@ public class FetchHTTPTest extends ProcessorTestBase { // server for digest auth server = new Server(); - sc = new SocketConnector(); + sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7778); server.addConnector(sc); @@ -325,7 +312,9 @@ public class FetchHTTPTest extends ProcessorTestBase { authWrapper = makeAuthWrapper(new DigestAuthenticator(), DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN, DIGEST_AUTH_PASSWORD); - authWrapper.setHandler(handlers); + HandlerCollection handlers2 = new HandlerCollection(); + handlers2.addHandler(new TestHandler()); + authWrapper.setHandler(handlers2); server.setHandler(authWrapper); server.start(); @@ -333,7 +322,7 @@ public class FetchHTTPTest extends ProcessorTestBase { return servers; } - + protected static void ensureHttpServers() throws Exception { if (httpServers == null) { httpServers = startHttpServers(); @@ -362,9 +351,4 @@ public class FetchHTTPTest extends ProcessorTestBase { } }; } - - public static Request getLastRequest() { - return lastRequest; - } - } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 43de8044..6ca6728c 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -157,7 +157,7 @@ public class FetchHTTPTests extends ProcessorTestBase { assertEquals(DEFAULT_PAYLOAD_STRING, curi.getRecorder().getContentReplayCharSequence().toString()); if (!exclusions.contains("httpBindAddress")) { - assertEquals("127.0.0.1", FetchHTTPTest.getLastRequest().getRemoteAddr()); + assertTrue(rawResponseString(curi).contains("Client-Host: 127.0.0.1\r\n")); } if (!exclusions.contains("nonFatalFailuresIsEmpty")) { @@ -262,7 +262,7 @@ public class FetchHTTPTests extends ProcessorTestBase { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); - assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); + assertEquals("basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); assertTrue(curi.getCredentials().contains(basicAuthCredential)); assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic")); @@ -406,8 +406,8 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); // the client bind address isn't recorded anywhere in heritrix as - // far as i can tell, so we get it this way... - assertEquals(addr, FetchHTTPTest.getLastRequest().getRemoteAddr()); + // far as i can tell, so we get the server to echo it back to us... + assertTrue(rawResponseString(curi).contains("Client-Host: " + addr + "\r\n")); runDefaultChecks(curi, "httpBindAddress"); } @@ -780,7 +780,9 @@ public class FetchHTTPTests extends ProcessorTestBase { fetcher().process(curi); // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertTrue(httpRequestString(curi).startsWith("GET /99% HTTP/1.0\r\n")); - runDefaultChecks(curi, "requestLine"); + // jetty 9 rejects requests with paths like this with 400 Bad Request + // so we can't run these checks anymore + //runDefaultChecks(curi, "requestLine"); } public void testTwoQuestionMarks() throws Exception { diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index ea2980e0..5e40d664 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -70,11 +70,11 @@ import org.archive.spring.ConfigPath; import org.archive.util.Base32; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.servlet.SessionHandler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; public class ContentDigestHistoryTest extends TmpDirTestCase { @@ -237,7 +237,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { fetcher.process(curi1); assertEquals(200, curi1.getFetchStatus()); - assertEquals(141, curi1.getContentSize()); assertEquals(expectedDigest, curi1.getContentDigestSchemeString()); assertFalse(curi1.hasContentDigestHistory()); @@ -260,7 +259,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { fetcher.process(curi2); assertEquals(200, curi1.getFetchStatus()); - assertEquals(141, curi1.getContentSize()); assertEquals(expectedDigest, curi1.getContentDigestSchemeString()); assertFalse(curi2.hasContentDigestHistory()); @@ -305,7 +303,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { assertTrue(recordIterator.hasNext()); record = recordIterator.next(); assertEquals(WARCRecordType.response.toString(), record.getHeader().getHeaderValue(HEADER_KEY_TYPE)); - assertEquals("141", record.getHeader().getHeaderValue(CONTENT_LENGTH)); assertEquals(expectedDigest, record.getHeader().getHeaderValue(HEADER_KEY_PAYLOAD_DIGEST)); assertEquals(curi1.getUURI().toString(), record.getHeader().getHeaderValue(HEADER_KEY_URI)); assertEquals(payloadRecordIdWithBrackets, record.getHeader().getHeaderValue(HEADER_KEY_ID)); @@ -374,10 +371,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { HandlerCollection handlers = new HandlerCollection(); handlers.addHandler(new SessionHandler(){ @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); @@ -388,7 +382,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { Server server = new Server(); server.setHandler(handlers); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); From 0782fb0481d38054f6530b9d3ef8a8745888445f Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 6 Aug 2019 15:10:48 +0900 Subject: [PATCH 09/23] JDK11: upgrade engine to jetty 9.4.19 and restlet 2.4.0 Jetty 9.4.12+ is required for TLS to work correctly under JDK11 (due to SSL handshake failures). In order to upgrade Jetty we also need to upgrade Restlet. There is one intentional change in behaviour to simplify upgrading. We remove a workaround for an old [webkit bug] where the browser claimed to prefer application/xml. The bug was fixed in 2011. [webkit bug]: https://bugs.webkit.org/show_bug.cgi?id=27267 Summary of Jetty API changes: - package names changed such as org.mortbay -> org.eclipse - SocketConnector and SslSocketConnector merged to ServerConnector - HashUserRealm split into UserStore and HashLoginService - SecurityHandler -> ConstraintSecurityHandler Summary of Restlet API changes: - some classes have moved package (Request, Response, Router etc) - ServerResource replaces Resource - represent(), acceptRepresentation() renamed to get(), post() - constructors were replaced by an init() method - setModifiable() was removed - getRequest().getEntityAsForm() -> new Form(entity) - Guard -> ChallengeAuthenticator Fixes #275 --- commons/src/test/resources/log4j.xml | 2 +- contrib/src/main/resources/log4j.xml | 2 +- contrib/src/test/resources/log4j.xml | 2 +- dist/src/main/conf/logging.properties | 2 +- dist/src/test/resources/log4j.xml | 2 +- engine/pom.xml | 62 ++++++++++--------- .../java/org/archive/crawler/Heritrix.java | 33 ++++++---- .../archive/crawler/restlet/BaseResource.java | 45 +------------- .../crawler/restlet/BeanBrowseResource.java | 37 ++++++----- .../crawler/restlet/EditRepresentation.java | 8 +-- .../crawler/restlet/EngineApplication.java | 19 +++--- .../crawler/restlet/EngineResource.java | 37 ++++++----- .../archive/crawler/restlet/EnhDirectory.java | 38 +++++++----- .../crawler/restlet/EnhDirectoryResource.java | 47 ++++++-------- .../org/archive/crawler/restlet/Flash.java | 4 +- .../crawler/restlet/JobRelatedResource.java | 23 +++---- .../archive/crawler/restlet/JobResource.java | 29 +++++---- .../crawler/restlet/PagedRepresentation.java | 8 +-- .../crawler/restlet/RateLimitGuard.java | 24 +++---- .../crawler/restlet/ReportGenResource.java | 24 +++---- .../crawler/restlet/ScriptResource.java | 31 +++++----- .../crawler/restlet/XmlMarshaller.java | 2 +- .../crawler/selftest/CheckpointSelfTest.java | 10 +-- .../crawler/selftest/FormAuthSelfTest.java | 23 +++---- .../crawler/selftest/FormLoginSelfTest.java | 24 +++---- .../crawler/selftest/HttpAuthSelfTest.java | 41 ++++++------ .../crawler/selftest/SelfTestBase.java | 14 ++--- .../crawler/selftest/UserAgentSelfTest.java | 22 +++---- .../archive/modules/fetcher/FormAuthTest.java | 60 +++++++++--------- engine/src/test/resources/log4j.xml | 2 +- modules/src/test/resources/log4j.xml | 2 +- 31 files changed, 328 insertions(+), 351 deletions(-) diff --git a/commons/src/test/resources/log4j.xml b/commons/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/commons/src/test/resources/log4j.xml +++ b/commons/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/contrib/src/main/resources/log4j.xml b/contrib/src/main/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/contrib/src/main/resources/log4j.xml +++ b/contrib/src/main/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/contrib/src/test/resources/log4j.xml b/contrib/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/contrib/src/test/resources/log4j.xml +++ b/contrib/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/dist/src/main/conf/logging.properties b/dist/src/main/conf/logging.properties index f8372396..094cb176 100644 --- a/dist/src/main/conf/logging.properties +++ b/dist/src/main/conf/logging.properties @@ -5,7 +5,7 @@ # ...and even less from the too-chatty-with-WARNINGs HttpClient library... org.apache.commons.httpclient.level = SEVERE org.restlet.Component.LogFilter.level = SEVERE -org.mortbay.log.level = SEVERE +org.eclipse.jetty.log.level = SEVERE # ...but INFO for our classes, which reserve FINE/FINER/FINEST for bulk/trivia... org.archive.level = INFO diff --git a/dist/src/test/resources/log4j.xml b/dist/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/dist/src/test/resources/log4j.xml +++ b/dist/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/engine/pom.xml b/engine/pom.xml index 14410e9e..909bdeb4 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -25,44 +25,50 @@ ${project.version} compile + - org.mortbay.jetty - jetty - 6.1.26 - compile + org.eclipse.jetty + jetty-server + 9.4.19.v20190610 - org.mortbay.jetty + org.eclipse.jetty jetty-util - 6.1.26 - compile + 9.4.19.v20190610 - org.mortbay.jetty - jetty-sslengine - 6.1.26 - compile + org.eclipse.jetty + jetty-servlet + 9.4.19.v20190610 + test + + + org.restlet.jse + org.restlet + 2.4.0 + + + org.restlet.jse + org.restlet.ext.jetty + 2.4.0 + + + org.eclipse.jetty + jetty-client + + - org.mortbay.jetty - jetty-ajp - 6.1.26 - compile + org.restlet.jse + org.restlet.ext.xml + 2.4.0 - org.restlet - org.restlet - 1.1.10 - - - com.noelios.restlet - com.noelios.restlet - 1.1.10 - - - com.noelios.restlet - com.noelios.restlet.ext.jetty - 1.1.10 + + org.restlet.jse + org.restlet.ext.crypto + 2.4.0 joda-time diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 11495072..a737f1d7 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -57,10 +57,11 @@ import org.archive.crawler.restlet.RateLimitGuard; import org.archive.util.ArchiveUtils; import org.archive.util.KeyTool; import org.restlet.Component; -import org.restlet.Guard; import org.restlet.Server; import org.restlet.data.ChallengeScheme; import org.restlet.data.Protocol; +import org.restlet.security.ChallengeAuthenticator; +import org.restlet.security.MapVerifier; /** @@ -212,7 +213,7 @@ public class Heritrix { "mailto, clsid, res, file, rtsp, about"); } - String maxFormSize = "org.mortbay.jetty.Request.maxFormContentSize"; + String maxFormSize = "org.eclipse.jetty.server.Request.maxFormContentSize"; if (System.getProperty(maxFormSize) == null) { System.setProperty(maxFormSize, "52428800"); } @@ -334,24 +335,30 @@ public class Heritrix { try { engine = new Engine(jobsDir); component = new Component(); - + if(bindHosts.isEmpty()) { // listen all addresses - setupServer(port, null, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, null, keystorePath, keystorePassword, keyPassword); } else { // bind only to declared addresses, or just 'localhost' for(String address : bindHosts) { - setupServer(port, address, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, address, keystorePath, keystorePassword, keyPassword); } } component.getClients().add(Protocol.FILE); - component.getClients().add(Protocol.CLAP); - Guard guard = new RateLimitGuard(null, + component.getClients().add(Protocol.CLAP); + + MapVerifier verifier = new MapVerifier(); + verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); + + ChallengeAuthenticator guard = new RateLimitGuard(component.getContext(), ChallengeScheme.HTTP_DIGEST, "Authentication Required"); - guard.getSecrets().put(authLogin, authPassword.toCharArray()); - component.getDefaultHost().attach(guard); + guard.setVerifier(verifier); guard.setNext(new EngineApplication(engine)); + + component.getDefaultHost().attach(guard); component.start(); + startupOut.println("engine listening at port "+port); startupOut.println("operator login set per " + ((aOption.startsWith("@")) ? "file "+aOption : "command-line")); @@ -457,16 +464,16 @@ public class Heritrix { /** * Create an HTTPS restlet Server instance matching the given parameters. - * + * + * @param component * @param port * @param address * @param keystorePath * @param keystorePassword * @param keyPassword */ - protected void setupServer(int port, String address, String keystorePath, String keystorePassword, String keyPassword) { - Server server = new Server(Protocol.HTTPS,address,port,null); - component.getServers().add(server); + protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) { + Server server = component.getServers().add(Protocol.HTTPS, address, port); server.getContext().getParameters().add("keystorePath", keystorePath); server.getContext().getParameters().add("keystorePassword", keystorePassword); server.getContext().getParameters().add("keyPassword", keyPassword); diff --git a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java index 1ebc80ef..72ae434b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java @@ -19,55 +19,14 @@ package org.archive.crawler.restlet; -import java.util.List; - -import org.restlet.Context; -import org.restlet.data.MediaType; -import org.restlet.data.Preference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Resource; -import org.restlet.resource.Variant; +import org.restlet.resource.ServerResource; /** * Abstract {@code Resource} with common shared functionality. * * @author nlevitt */ -public abstract class BaseResource extends Resource { - - public BaseResource(Context ctx, Request req, Response res) { - super(ctx, req, res); - } - - /** - * If client can accept text/html, always prefer it. WebKit-based browsers - * claim to want application/xml, but we don't want to give it to them. See - * https://webarchive.jira.com/browse/HER-1603 - */ - public Variant getPreferredVariant() { - boolean addExplicitTextHtmlPreference = false; - - for (Preference mediaTypePreference: getRequest().getClientInfo().getAcceptedMediaTypes()) { - if (mediaTypePreference.getMetadata().equals(MediaType.TEXT_HTML)) { - mediaTypePreference.setQuality(Float.MAX_VALUE); - addExplicitTextHtmlPreference = false; - break; - } else if (mediaTypePreference.getMetadata().includes(MediaType.TEXT_HTML)) { - addExplicitTextHtmlPreference = true; - } - } - - if (addExplicitTextHtmlPreference) { - List> acceptedMediaTypes = getRequest().getClientInfo().getAcceptedMediaTypes(); - acceptedMediaTypes.add(new Preference(MediaType.TEXT_HTML, Float.MAX_VALUE)); - getRequest().getClientInfo().setAcceptedMediaTypes(acceptedMediaTypes); - } - - - return super.getPreferredVariant(); - } - +public abstract class BaseResource extends ServerResource { protected String getStaticRef(String resource) { String rootRef = getRequest().getRootRef().toString(); return rootRef + "/engine/static/" + resource; diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index b09d70af..5ae0c897 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -19,16 +19,12 @@ package org.archive.crawler.restlet; -import java.io.File; import java.io.IOException; -import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.net.URLDecoder; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Set; @@ -36,18 +32,18 @@ import org.apache.commons.lang.StringUtils; import org.archive.crawler.restlet.models.BeansModel; import org.archive.crawler.restlet.models.ViewModel; import org.archive.spring.PathSharingContext; -import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import org.springframework.beans.BeanWrapperImpl; import org.springframework.beans.BeansException; @@ -69,12 +65,12 @@ public class BeanBrowseResource extends JobRelatedResource { protected PathSharingContext appCtx; protected String beanPath; private Configuration _templateConfiguration; - - public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); - setModifiable(true); // accept POSTs appCtx = cj.getJobContext(); beanPath = (String)req.getAttributes().get("beanPath"); if (beanPath!=null) { @@ -99,13 +95,14 @@ public class BeanBrowseResource extends JobRelatedResource { return _templateConfiguration; } - public void acceptRepresentation(Representation entity) throws ResourceException { + @Override + protected Representation post(Representation entity, Variant variant) throws ResourceException { if (appCtx == null) { throw new ResourceException(404); } // copy op? - Form form = getRequest().getEntityAsForm(); + Form form = new Form(entity); beanPath = form.getFirstValue("beanPath"); String newVal = form.getFirstValue("newVal"); @@ -122,6 +119,7 @@ public class BeanBrowseResource extends JobRelatedResource { ref.setPath(getBeansRefPath()); ref.addSegment(beanPath); getResponse().redirectSeeOther(ref); + return new EmptyRepresentation(); } public String getBeansRefPath() { @@ -137,7 +135,8 @@ public class BeanBrowseResource extends JobRelatedResource { return path; } - public Representation represent(Variant variant) throws ResourceException { + @Override + public Representation get(Variant variant) throws ResourceException { if (appCtx == null) { throw new ResourceException(404); } @@ -209,8 +208,8 @@ public class BeanBrowseResource extends JobRelatedResource { for(String name: appCtx.getBeanDefinitionNames()) { addPresentableNestedNames(nestedNames, appCtx.getBean(name), alreadyWritten); } - - return new BeansModel(cj.getShortName(), + + return new BeansModel(cj.getShortName(), new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef().toString(), beanPath, bean, diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index a4ea7996..232dbdd2 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -30,8 +30,8 @@ import org.apache.commons.lang.StringEscapeUtils; import org.restlet.data.CharacterSet; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.resource.CharacterRepresentation; -import org.restlet.resource.FileRepresentation; +import org.restlet.representation.CharacterRepresentation; +import org.restlet.representation.FileRepresentation; /** * Representation wrapping a FileRepresentation, displaying its contents @@ -40,9 +40,9 @@ import org.restlet.resource.FileRepresentation; * @author gojomo */ public class EditRepresentation extends CharacterRepresentation { - protected FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; protected EnhDirectoryResource dirResource; - + public EditRepresentation(FileRepresentation representation, EnhDirectoryResource resource) { super(MediaType.TEXT_HTML); fileRepresentation = representation; diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java index 6c7793f7..a82c3026 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java @@ -26,19 +26,19 @@ import java.io.StringWriter; import org.archive.crawler.framework.Engine; import org.archive.util.TextUtils; import org.restlet.Application; -import org.restlet.Directory; -import org.restlet.Redirector; import org.restlet.Restlet; -import org.restlet.Router; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.Status; -import org.restlet.resource.Representation; -import org.restlet.resource.StringRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; +import org.restlet.resource.Directory; +import org.restlet.routing.Redirector; +import org.restlet.routing.Router; +import org.restlet.routing.Template; import org.restlet.service.StatusService; -import org.restlet.util.Template; /** * Restlet Application for a Heritrix crawl 'Engine', which is aware of @@ -56,7 +56,8 @@ public class EngineApplication extends Application { setStatusService(new EngineStatusService()); } - public synchronized Restlet createRoot() { + @Override + public Restlet createInboundRoot() { Router router = new Router(getContext()); router.attach("/",new Redirector(null,"/engine",Redirector.MODE_CLIENT_TEMPORARY)); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index b18d9f3d..dabd7f3e 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -30,21 +30,24 @@ import org.archive.crawler.framework.Engine; import org.archive.crawler.restlet.models.EngineModel; import org.archive.crawler.restlet.models.ViewModel; import org.restlet.Context; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; +import static org.restlet.data.MediaType.APPLICATION_XML; + /** * Restlet Resource representing an Engine that may be used * to assemble, launch, monitor, and manage crawls. @@ -56,11 +59,12 @@ import freemarker.template.TemplateException; public class EngineResource extends BaseResource { private Configuration _templateConfiguration; - public EngineResource(Context ctx, Request req, Response res) { - super(ctx, req, res); - setModifiable(true); + + @Override + public void init(Context ctx, Request req, Response res) { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); - getVariants().add(new Variant(MediaType.APPLICATION_XML)); + getVariants().add(new Variant(APPLICATION_XML)); Configuration tmpltCfg = new Configuration(); tmpltCfg.setClassForTemplateLoading(this.getClass(),""); @@ -74,10 +78,12 @@ public class EngineResource extends BaseResource { public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public Representation represent(Variant variant) throws ResourceException { + + @Override + protected Representation get(Variant variant) throws ResourceException { Representation representation; - if (variant.getMediaType() == MediaType.APPLICATION_XML) { - representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + if (variant.getMediaType() == APPLICATION_XML) { + representation = new WriterRepresentation(APPLICATION_XML) { public void write(Writer writer) throws IOException { XmlMarshaller.marshalDocument(writer, "engine", makeDataModel()); } @@ -93,10 +99,10 @@ public class EngineResource extends BaseResource { representation.setCharacterSet(CharacterSet.UTF_8); return representation; } - + @Override - public void acceptRepresentation(Representation entity) throws ResourceException { - Form form = getRequest().getEntityAsForm(); + protected Representation post(Representation entity, Variant variant) throws ResourceException { + Form form = new Form(entity); String action = form.getFirstValue("action"); if("rescan".equals(action)) { getEngine().findJobConfigs(); @@ -182,6 +188,7 @@ public class EngineResource extends BaseResource { } // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + return new EmptyRepresentation(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java index ea7bb79d..bb26b16b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java @@ -26,10 +26,12 @@ import java.util.logging.Level; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.restlet.Context; -import org.restlet.Handler; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.data.Status; +import org.restlet.resource.Directory; +import org.restlet.resource.ServerResource; /** * Enhanced version of Restlet Directory, which allows the local @@ -39,11 +41,11 @@ import org.restlet.data.Response; * * @author gojomo */ -public abstract class EnhDirectory extends org.restlet.Directory { +public abstract class EnhDirectory extends Directory { protected IOFileFilter editFilter = FileFilterUtils.falseFileFilter(); protected IOFileFilter pageFilter = FileFilterUtils.falseFileFilter(); protected IOFileFilter tailFilter = FileFilterUtils.falseFileFilter(); - + public EnhDirectory(Context context, Reference rootLocalReference) { super(context, rootLocalReference); // TODO Auto-generated constructor stub @@ -55,21 +57,27 @@ public abstract class EnhDirectory extends org.restlet.Directory { } @Override - public Handler findTarget(Request request, Response response) { - Handler retVal; - synchronized(this) { + public void handle(Request request, Response response) { + synchronized (this) { Reference oldRef = getRootRef(); setRootRef(determineRootRef(request)); try { - retVal = new EnhDirectoryResource(this, request, response); - } catch (IOException ioe) { - getLogger().log(Level.WARNING, - "Unable to find the directory's resource", ioe); - retVal = null; + super.handle(request, response); + } finally { + setRootRef(oldRef); + } + + // XXX: FileRepresentation.isAvailable() returns false for empty files generating status 204 No Content + // which confuses browsers. Force it back it 200 OK. + if (response.getStatus() == Status.SUCCESS_NO_CONTENT) { + response.setStatus(Status.SUCCESS_OK); } - setRootRef(oldRef); } - return retVal; + } + + @Override + public ServerResource create(Request request, Response response) { + return new EnhDirectoryResource(); } protected abstract Reference determineRootRef(Request request); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java index 5f7c0b5e..18c59354 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java @@ -23,22 +23,18 @@ package org.archive.crawler.restlet; import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import org.apache.commons.io.FileUtils; -import org.restlet.data.CharacterSet; -import org.restlet.data.Form; -import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.data.Status; -import org.restlet.resource.FileRepresentation; -import org.restlet.resource.Representation; +import org.restlet.data.*; +import org.restlet.engine.local.DirectoryServerResource; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.FileRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.Variant; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; - -import com.noelios.restlet.local.DirectoryResource; /** * Enhanced version of Restlet DirectoryResource, adding ability to @@ -46,20 +42,15 @@ import com.noelios.restlet.local.DirectoryResource; * * @author gojomo */ -public class EnhDirectoryResource extends DirectoryResource { - - public EnhDirectoryResource(EnhDirectory directory, Request request, Response response) throws IOException { - super(directory, request, response); - } - - /** +public class EnhDirectoryResource extends DirectoryServerResource { + /** * Add EditRepresentation as a variant when appropriate. * - * @see com.noelios.restlet.local.DirectoryResource#getVariants() + * @see org.restlet.engine.local.DirectoryServerResource#getVariants() */ @Override public List getVariants() { - List variants = super.getVariants(); + List variants = new LinkedList<>(super.getVariants(Method.GET)); Form f = getRequest().getResourceRef().getQueryAsForm(); String format = f.getFirstValue("format"); if("textedit".equals(format)) { @@ -73,7 +64,7 @@ public class EnhDirectoryResource extends DirectoryResource { } catch (Exception e) { throw new RuntimeException(e); } - variants = super.getVariants(); + variants = new LinkedList<>(super.getVariants(Method.GET)); } // wrap FileRepresentations in EditRepresentations ListIterator iter = variants.listIterator(); @@ -119,18 +110,18 @@ public class EnhDirectoryResource extends DirectoryResource { } protected EnhDirectory getEnhDirectory() { - return (EnhDirectory)getDirectory(); + return (EnhDirectory) getDirectory(); } - /** + /** * Accept a POST used to edit or create a file. * - * @see org.restlet.resource.Resource#acceptRepresentation(org.restlet.resource.Representation) + * @see org.restlet.resource.ServerResource#post(Representation, Variant) */ - public void acceptRepresentation(Representation entity) - throws ResourceException { + @Override + protected Representation post(Representation entity, Variant variant) throws ResourceException { // TODO: only allowPost on valid targets - Form form = getRequest().getEntityAsForm(); + Form form = new Form(entity); String newContents = form.getFirstValue("contents"); EditRepresentation er; try { @@ -152,6 +143,6 @@ public class EnhDirectoryResource extends DirectoryResource { Reference ref = getRequest().getOriginalRef().clone(); /// ref.setQuery(null); getResponse().redirectSeeOther(ref); - + return new EmptyRepresentation(); } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/Flash.java b/engine/src/main/java/org/archive/crawler/restlet/Flash.java index a43d6c9e..44140866 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/Flash.java +++ b/engine/src/main/java/org/archive/crawler/restlet/Flash.java @@ -29,10 +29,10 @@ import java.util.Map.Entry; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.RandomUtils; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.Cookie; import org.restlet.data.CookieSetting; -import org.restlet.data.Request; -import org.restlet.data.Response; import org.restlet.util.Series; /** diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index ae967a0a..4dfe548c 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -40,8 +40,8 @@ import org.archive.crawler.framework.Engine; import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.resource.ResourceException; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanWrapperImpl; @@ -58,18 +58,19 @@ public abstract class JobRelatedResource extends BaseResource { private final static Logger LOGGER = Logger.getLogger(JobRelatedResource.class.getName()); - protected CrawlJob cj; + protected CrawlJob cj; protected IdentityHashMap beanToNameMap; - - public JobRelatedResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) { + super.init(ctx, req, res); cj = getEngine().getJob((String)req.getAttributes().get("job")); - if(cj==null) { + if(cj == null) { throw new ResourceException(404); } } - + protected Engine getEngine() { return ((EngineApplication)getApplication()).getEngine(); } @@ -95,13 +96,13 @@ public abstract class JobRelatedResource extends BaseResource { return; } - - Reference baseRef = getRequest().getResourceRef().getBaseRef(); + + Reference baseRef = getRequest().getResourceRef().getBaseRef(); if (baseRef.getPath().endsWith("beans")) { baseRef.setPath(baseRef.getPath() + "/"); } - + if (getBeanToNameMap().containsKey(obj)) { // this object is itself a named bean Map bean = new LinkedHashMap(); diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index d3b0a295..5b4ab6c7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -38,12 +38,13 @@ import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.ObjectWrapper; @@ -68,10 +69,10 @@ public class JobResource extends BaseResource { protected CrawlJob cj; - public JobResource(Context ctx, Request req, Response res) + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); - setModifiable(true); + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); cj = getEngine().getJob( @@ -88,7 +89,9 @@ public class JobResource extends BaseResource { public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public Representation represent(Variant variant) throws ResourceException { + + @Override + public Representation get(Variant variant) throws ResourceException { if (cj == null) { throw new ResourceException(404); } @@ -184,19 +187,18 @@ public class JobResource extends BaseResource { } @Override - public void acceptRepresentation(Representation entity) + public Representation post(Representation entity, Variant variant) throws ResourceException { if (cj == null) { throw new ResourceException(404); } // copy op? - Form form = null; - form = getRequest().getEntityAsForm(); + Form form = new Form(entity); String copyTo = form.getFirstValue("copyTo"); if (copyTo != null) { copyJob(copyTo, "on".equals(form.getFirstValue("asProfile"))); - return; + return new EmptyRepresentation(); } AlertHandler.ensureStaticInitialization(); AlertThreadGroup.setThreadLogger(cj.getJobLogger()); @@ -241,6 +243,7 @@ public class JobResource extends BaseResource { // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + return new EmptyRepresentation(); } protected void copyJob(String copyTo, boolean asProfile) diff --git a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java index 363ee674..aabd0d46 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java @@ -38,8 +38,8 @@ import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.resource.CharacterRepresentation; -import org.restlet.resource.FileRepresentation; +import org.restlet.representation.CharacterRepresentation; +import org.restlet.representation.FileRepresentation; /** * Representation wrapping a FileRepresentation, displaying its contents @@ -50,7 +50,7 @@ import org.restlet.resource.FileRepresentation; public class PagedRepresentation extends CharacterRepresentation { // passed-in at construction /** wrapped FileRepresentation **/ - protected FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; /** wrapped EnhDirectoryResource; used to formulate self-links **/ protected EnhDirectoryResource dirResource; @@ -124,7 +124,7 @@ public class PagedRepresentation extends CharacterRepresentation { /** * Write the paged HTML. * - * @see org.restlet.resource.Representation#write(java.io.Writer) + * @see org.restlet.representation.Representation#write(java.io.Writer) */ @Override public void write(Writer writer) throws IOException { diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index d4d304ca..042fd2ea 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -18,21 +18,21 @@ */ package org.archive.crawler.restlet; -import java.util.Collection; import java.util.logging.Logger; import org.restlet.Context; -import org.restlet.Guard; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.ChallengeScheme; -import org.restlet.data.Request; +import org.restlet.security.ChallengeAuthenticator; /** - * Guard that slows and logs failed authentication attempts, to make + * ChallengeAuthenticator that slows and logs failed authentication attempts, to make * brute-force guessing attacks less feasible. * * @author gojomo */ -public class RateLimitGuard extends Guard { +public class RateLimitGuard extends ChallengeAuthenticator { private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000; private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); @@ -43,14 +43,10 @@ public class RateLimitGuard extends Guard { super(context, scheme, realm); } - public RateLimitGuard(Context context, String realm, Collection baseUris, String serverKey) { - super(context, realm, baseUris, serverKey); - } - @Override - public synchronized int authenticate(Request request) { - int retVal = super.authenticate(request); - if(retVal == AUTHENTICATION_INVALID) { + protected boolean authenticate(Request request, Response response) { + boolean succeeded = super.authenticate(request, response); + if (!succeeded) { logger.warning("authentication failure "+request); // wait until at least LAG has passed from last failure // holding object lock the whole time, so no other checks @@ -64,8 +60,8 @@ public class RateLimitGuard extends Guard { // ignore } } - lastFailureTime = now + sleepMs; + lastFailureTime = now + sleepMs; } - return retVal; + return succeeded; } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java index 3ccc8794..5468bf55 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java @@ -23,12 +23,12 @@ import java.io.File; import org.restlet.Context; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.StringRepresentation; -import org.restlet.resource.Variant; +import org.restlet.representation.Variant; /** * Restlet Resource which generates fresh reports and then redirects @@ -38,24 +38,26 @@ import org.restlet.resource.Variant; */ public class ReportGenResource extends JobRelatedResource { protected String reportClass; - - public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_PLAIN)); reportClass = (String)req.getAttributes().get("reportClass"); } - public Representation represent(Variant variant) throws ResourceException { + @Override + protected Representation get(Variant variant) throws ResourceException { // generate report if (cj == null || cj.getCrawlController() == null) { throw new ResourceException(500); } - File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); + File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); if (f==null) { throw new ResourceException(500); } // redirect - String relative = JobResource.getHrefPath(f,cj); + String relative = JobResource.getHrefPath(f, cj); if(relative!=null) { getResponse().redirectSeeOther("../"+relative+"?m="+f.lastModified()); return new StringRepresentation(""); diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 5a8bfa6b..58fee0f9 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -41,12 +41,12 @@ import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.ObjectWrapper; @@ -81,10 +81,10 @@ public class ScriptResource extends JobRelatedResource { protected String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0); private Configuration _templateConfiguration; - - public ScriptResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); - setModifiable(true); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); @@ -92,7 +92,7 @@ public class ScriptResource extends JobRelatedResource { tmpltCfg.setClassForTemplateLoading(this.getClass(),""); tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER); setTemplateConfiguration(tmpltCfg); - + scriptingConsole = new ScriptingConsole(cj); } public void setTemplateConfiguration(Configuration tmpltCfg) { @@ -105,8 +105,8 @@ public class ScriptResource extends JobRelatedResource { private ScriptingConsole scriptingConsole; @Override - public void acceptRepresentation(Representation entity) throws ResourceException { - Form form = getRequest().getEntityAsForm(); + public Representation post(Representation entity, Variant variant) throws ResourceException { + Form form = new Form(entity); chosenEngine = form.getFirstValue("engine"); String script = form.getFirstValue("script"); if(StringUtils.isBlank(script)) { @@ -121,10 +121,11 @@ public class ScriptResource extends JobRelatedResource { //TODO: log script, results somewhere; job log INFO? - getResponse().setEntity(represent()); + return get(variant); } - - public Representation represent(Variant variant) throws ResourceException { + + @Override + public Representation get(Variant variant) throws ResourceException { Representation representation; if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { diff --git a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java index 11098b38..176204c6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java +++ b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java @@ -36,7 +36,7 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; -import org.restlet.util.XmlWriter; +import org.restlet.ext.xml.XmlWriter; import org.xml.sax.SAXException; /** diff --git a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java index 138f47a1..7f7d9e6b 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java @@ -22,10 +22,10 @@ package org.archive.crawler.selftest; import java.io.IOException; import org.archive.crawler.framework.CrawlJob; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; /** @@ -85,7 +85,7 @@ public class CheckpointSelfTest extends SelfTestBase { private Server makeHttpServer(int port) throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost(HOST); sc.setPort(port); server.addConnector(sc); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java index 63d4b764..6e04f56a 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java @@ -19,19 +19,20 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; + import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; /** * Test form-based authentication @@ -60,17 +61,17 @@ public class FormAuthSelfTest protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java index 510fb839..332f6835 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java @@ -19,20 +19,20 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; + import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; - /** * Test form-based authentication * @@ -60,17 +60,17 @@ public class FormLoginSelfTest protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java index a0f68464..6ad5ebd2 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java @@ -24,17 +24,16 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.servlet.ServletHandler; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; /** * Test HTTP basic authentication @@ -73,27 +72,27 @@ public class HttpAuthSelfTest ConstraintMapping cm = new ConstraintMapping(); cm.setConstraint(constraint); cm.setPathSpec("/basic/*"); + + UserStore userStore = new UserStore(); + userStore.addUser("Mr. Happy Pants", new Password("xyzzy"), new String[]{"rule"}); + HashLoginService loginService = new HashLoginService("Hyrule"); + loginService.setUserStore(userStore); - HashUserRealm realm = new HashUserRealm(); - realm.setName("Hyrule"); - realm.put("Mr. Happy Pants", "xyzzy"); - realm.addUserToRole("Mr. Happy Pants", "user"); - - SecurityHandler securityHandler = new SecurityHandler(); - securityHandler.setUserRealm(realm); + ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler(); + securityHandler.setLoginService(loginService); securityHandler.setConstraintMappings(new ConstraintMapping[]{cm}); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { securityHandler, rhandler, servletHandler, diff --git a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java index 0771360c..7f09dfbe 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java +++ b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java @@ -38,12 +38,12 @@ import org.archive.io.arc.ARCReaderFactory; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; /** * Base class for 'self tests', integrations tests formatted as unit @@ -182,7 +182,7 @@ public abstract class SelfTestBase extends TmpDirTestCase { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); diff --git a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java index 48925345..d70ddc0b 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java @@ -20,14 +20,14 @@ package org.archive.crawler.selftest; import org.archive.util.ArchiveUtils; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; /** * @author pjack @@ -54,17 +54,17 @@ public class UserAgentSelfTest extends SelfTestBase { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java index 7d3ce576..0677cbc5 100644 --- a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java +++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java @@ -44,20 +44,19 @@ import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.NCSARequestLog; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.handler.RequestLogHandler; -import org.mortbay.jetty.security.Authenticator; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.FormAuthenticator; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.servlet.HashSessionManager; -import org.mortbay.jetty.servlet.SessionHandler; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.security.authentication.FormAuthenticator; +import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.handler.RequestLogHandler; +import org.eclipse.jetty.server.session.DefaultSessionCache; +import org.eclipse.jetty.server.session.SessionCache; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; /* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but * the code is written, it's easier to run in eclipse, and no doubt tests @@ -211,12 +210,9 @@ public class FormAuthTest extends TestCase { public FormAuthTestHandler() { super(); } - + @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (target.endsWith("/set-cookie")) { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } @@ -247,16 +243,16 @@ public class FormAuthTest extends TestCase { constraintMapping.setConstraint(constraint); constraintMapping.setPathSpec("/auth/*"); - SecurityHandler authWrapper = new SecurityHandler(); + UserStore userStore = new UserStore(); + userStore.addUser(login, new Password(password), new String[]{role}); + + HashLoginService loginService = new HashLoginService(realm); + loginService.setUserStore(userStore); + + ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); - authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); - authWrapper.setUserRealm(new HashUserRealm(realm) { - { - put(login, password); - addUserToRole(login, role); - } - }); + authWrapper.setLoginService(loginService); return authWrapper; } @@ -265,7 +261,7 @@ public class FormAuthTest extends TestCase { // server for form auth Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7779); server.addConnector(sc); @@ -277,16 +273,16 @@ public class FormAuthTest extends TestCase { requestLogHandler.setRequestLog(requestLog); handlers.addHandler(requestLogHandler); - FormAuthenticator formAuthenticatrix = new FormAuthenticator(); - formAuthenticatrix.setLoginPage("/login.html"); - + FormAuthenticator formAuthenticatrix = new FormAuthenticator("/login.html", null, false); + SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix, FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN, FORM_AUTH_PASSWORD); authWrapper.setHandler(handlers); SessionHandler sessionHandler = new SessionHandler(); - sessionHandler.setSessionManager(new HashSessionManager()); + SessionCache cache = new DefaultSessionCache(sessionHandler); + sessionHandler.setSessionCache(cache); sessionHandler.setHandler(authWrapper); server.setHandler(sessionHandler); diff --git a/engine/src/test/resources/log4j.xml b/engine/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/engine/src/test/resources/log4j.xml +++ b/engine/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/modules/src/test/resources/log4j.xml b/modules/src/test/resources/log4j.xml index 475b130e..e04f34ab 100644 --- a/modules/src/test/resources/log4j.xml +++ b/modules/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + From 996d1366b096a6f72f27843ac98ef7130988c770 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 6 Aug 2019 16:41:09 +0900 Subject: [PATCH 10/23] Drop support for JDK 7 Jetty 9.4 requires JDK 8 or later. JDK 11 requires Jetty 9.4.12+. Therefore unfortunately we cannot easily support both JDK 11 and JDK 7 at the same time. --- .travis.yml | 7 ++----- pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20978aa4..c3f0a33a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,6 @@ matrix: include: - jdk: oraclejdk8 dist: trusty - - jdk: openjdk7 - dist: trusty - env: PROJECTS='--projects commons,modules,engine' - jdk: openjdk8 - jdk: openjdk11 allow_failures: @@ -22,14 +19,14 @@ before_install: - "export _JAVA_OPTIONS=-Xmx1500m" - "echo _JAVA_OPTIONS=$_JAVA_OPTIONS" -install: mvn dependency:resolve -B -V $PROJECTS +install: mvn dependency:resolve -B -V cache: directories: - $HOME/.m2 script: - - travis_wait 30 mvn install $PROJECTS + - travis_wait 30 mvn install after_failure: - cat */target/surefire-reports/*.txt diff --git a/pom.xml b/pom.xml index 033cd488..fd89cb59 100644 --- a/pom.xml +++ b/pom.xml @@ -373,8 +373,8 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html maven-compiler-plugin 3.3 - 1.7 - 1.7 + 1.8 + 1.8 UTF-8 From 4012a5f4dd392831f5057048ba5c62d985d74331 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 6 Aug 2019 16:56:10 +0900 Subject: [PATCH 11/23] Fix 'No session data store configured' test error --- .../src/test/java/org/archive/modules/fetcher/FormAuthTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java index 0677cbc5..39ca0bb6 100644 --- a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java +++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java @@ -53,6 +53,7 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.server.session.DefaultSessionCache; +import org.eclipse.jetty.server.session.NullSessionDataStore; import org.eclipse.jetty.server.session.SessionCache; import org.eclipse.jetty.server.session.SessionHandler; import org.eclipse.jetty.util.security.Constraint; @@ -282,6 +283,7 @@ public class FormAuthTest extends TestCase { SessionHandler sessionHandler = new SessionHandler(); SessionCache cache = new DefaultSessionCache(sessionHandler); + cache.setSessionDataStore(new NullSessionDataStore()); sessionHandler.setSessionCache(cache); sessionHandler.setHandler(authWrapper); server.setHandler(sessionHandler); From 6c2afc36b0830e6614db96d9b1c438fadbc1e914 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 6 Aug 2019 17:03:58 +0900 Subject: [PATCH 12/23] Update statistics assertions as Jetty 9 produces different headers --- .../crawler/selftest/StatisticsSelfTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index 8d880bf3..ceea9517 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -48,12 +48,12 @@ public class StatisticsSelfTest extends SelfTestBase { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); + assertEquals(7501, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(2133, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(5368, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); } @@ -66,17 +66,17 @@ public class StatisticsSelfTest extends SelfTestBase { sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(2942l, (long) sourceStats.get("novel")); + assertEquals(2133l, (long) sourceStats.get("novel")); assertEquals(3l, (long) sourceStats.get("novelCount")); - assertEquals(2942l, (long) sourceStats.get("warcNovelContentBytes")); + assertEquals(2133l, (long) sourceStats.get("warcNovelContentBytes")); assertEquals(3l, (long) sourceStats.get("warcNovelUrls")); sourceStats = stats.getSourceStats("http://localhost:7777/b.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); + assertEquals(5368l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); + assertEquals(5368l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); } From 97b9c3c91c9b2fdbe44d26a36627904e3e0422bf Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 6 Aug 2019 17:13:29 +0900 Subject: [PATCH 13/23] Fix restlet child component context warning --- engine/src/main/java/org/archive/crawler/Heritrix.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index a737f1d7..ee0fcf79 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -351,7 +351,7 @@ public class Heritrix { MapVerifier verifier = new MapVerifier(); verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); - ChallengeAuthenticator guard = new RateLimitGuard(component.getContext(), + ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(), ChallengeScheme.HTTP_DIGEST, "Authentication Required"); guard.setVerifier(verifier); guard.setNext(new EngineApplication(engine)); From 8b72be7d1da2f5099ed9b2af9d0f1eb538cdcf0a Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 8 Aug 2019 10:37:38 +0900 Subject: [PATCH 14/23] Now jdk11 support is merged we can disallow test failures on it --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3f0a33a..e234b918 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ matrix: dist: trusty - jdk: openjdk8 - jdk: openjdk11 - allow_failures: - - jdk: openjdk11 before_install: - "export JAVA_OPTS=-Xmx1500m" From 92ffdede0d017472e56dac64f622efdab5d3f3e2 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 15:03:52 +0900 Subject: [PATCH 15/23] Link to javadoc.io for more recent api docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1a47653..2dac370f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Heritrix [![Build Status](https://travis-ci.org/internetarchive/heritrix3.svg?branch=master)](https://travis-ci.org/internetarchive/heritrix3) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.archive/heritrix) -[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](http://builds.archive.org/javadoc/heritrix-3.2.0/) +[![Javadoc](https://javadoc-badge.appspot.com/org.archive/heritrix.svg?label=javadoc)](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine) [![LICENSE](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](./LICENSE) ## Introduction @@ -21,7 +21,7 @@ load your crawl will place on seed sites and set politeness policies accordingly - [Developer Manual](http://crawler.archive.org/articles/developer_manual/index.html) - [REST API documentation](https://heritrix.readthedocs.io/en/latest/api.html) -- [JavaDoc](http://builds.archive.org/javadoc/heritrix-3.2.0/) (n.b. Javadoc currently out of date) +- JavaDoc: [engine](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine), [modules](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-modules), [commons](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-commons), [contrib](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-contrib) ## Latest Releases From 36003b8626273fda6a643d719f2d973ef0393655 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 17:36:16 +0900 Subject: [PATCH 16/23] Fix digest authentication In Restlet 2 it appears we need to use DigestAuthenticator. (Previously both digest and basic auth were handled by the same Guard class.) --- .../src/main/java/org/archive/crawler/Heritrix.java | 6 +++--- .../org/archive/crawler/restlet/RateLimitGuard.java | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index ee0fcf79..5714b9f6 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -351,9 +351,9 @@ public class Heritrix { MapVerifier verifier = new MapVerifier(); verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); - ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(), - ChallengeScheme.HTTP_DIGEST, "Authentication Required"); - guard.setVerifier(verifier); + RateLimitGuard guard = new RateLimitGuard(component.getContext().createChildContext(), + "Authentication Required", UUID.randomUUID().toString()); + guard.setWrappedVerifier(verifier); guard.setNext(new EngineApplication(engine)); component.getDefaultHost().attach(guard); diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index 042fd2ea..7d9cd703 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -18,13 +18,12 @@ */ package org.archive.crawler.restlet; -import java.util.logging.Logger; - import org.restlet.Context; import org.restlet.Request; import org.restlet.Response; -import org.restlet.data.ChallengeScheme; -import org.restlet.security.ChallengeAuthenticator; +import org.restlet.ext.crypto.DigestAuthenticator; + +import java.util.logging.Logger; /** * ChallengeAuthenticator that slows and logs failed authentication attempts, to make @@ -32,15 +31,15 @@ import org.restlet.security.ChallengeAuthenticator; * * @author gojomo */ -public class RateLimitGuard extends ChallengeAuthenticator { +public class RateLimitGuard extends DigestAuthenticator { private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000; private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); protected long lastFailureTime = 0; - public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) throws IllegalArgumentException { - super(context, scheme, realm); + public RateLimitGuard(Context context, String realm, String serverKey) throws IllegalArgumentException { + super(context, realm, serverKey); } @Override From 38c44e8e8e6189bdd46e7ffeb80fa7657eb27fd5 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 17:50:35 +0900 Subject: [PATCH 17/23] Add missing UUID import (interactive commit fail) --- engine/src/main/java/org/archive/crawler/Heritrix.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 5714b9f6..dd9e208e 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -33,12 +33,7 @@ import java.net.UnknownHostException; import java.security.KeyStore; import java.security.MessageDigest; import java.security.cert.Certificate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; import java.util.logging.LogManager; import java.util.logging.Logger; From a5c03d9b73b000d61b8fc4874f9a90cc884a4937 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 12 Aug 2019 18:08:43 +0900 Subject: [PATCH 18/23] Mitigate random CookieStore.testConcurrentLoad test failures The arbitary value `25` was used but in prace it's quite possible for more than 25 writing threads to have checked the cookie count limit before adding their cookie. In practice we see Travis failing on this test quite often, every few builds in fact. I think using `threads.length` (i.e. 200) should cover the worst case possibility where every thread reads a stale count and tries to add their cookie. Fixes #274 --- .../java/org/archive/modules/fetcher/CookieStoreTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java index a621a57d..37632a04 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java @@ -388,7 +388,9 @@ public class CookieStoreTest extends TmpDirTestCase { } for (String domain: domainCounts.keySet()) { - assertTrue(domainCounts.get(domain) <= BdbCookieStore.MAX_COOKIES_FOR_DOMAIN + 25); + // the cookie store intentionally doesn't synchronize so we allow up to thread.length + // additional cookies over the limit + assertTrue(domainCounts.get(domain) <= BdbCookieStore.MAX_COOKIES_FOR_DOMAIN + threads.length); } } From 2a5bd87e5114ec4becde464a4b5a709fdaf1e625 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 16 Aug 2019 22:11:26 +0900 Subject: [PATCH 19/23] Remove testConcurrentLoad Noah wrote in #280: > Maybe we should just drop the test. The assumption when we wrote the > test was that a race condition would not be so frequent in practice. > We've seen that under the contrived conditions created by the test > case, it is frequent. But that's ok --- .../modules/fetcher/CookieStoreTest.java | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java index 37632a04..35ff08d0 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieStoreTest.java @@ -338,62 +338,7 @@ public class CookieStoreTest extends TmpDirTestCase { assertTrue(bdbCookieList.size() > 3000); assertCookieListsEquivalent(bdbCookieList, basicCookieStore().getCookies()); } - - public void testConcurrentLoad() throws IOException, InterruptedException { - bdbCookieStore().clear(); - basicCookieStore().clear(); - final Random rand = new Random(); - Runnable runnable = new Runnable() { - @Override - public void run() { - try { - while (!Thread.interrupted()) { - BasicClientCookie cookie = new BasicClientCookie(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - cookie.setDomain("d" + rand.nextInt(20) + ".example.com"); - bdbCookieStore().addCookie(cookie); - basicCookieStore().addCookie(cookie); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - - Thread[] threads = new Thread[200]; - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(runnable); - threads[i].setName("cookie-load-test-" + i); - threads[i].start(); - } - - Thread.sleep(1000); - - for (int i = 0; i < threads.length; i++) { - threads[i].interrupt(); - } - for (int i = 0; i < threads.length; i++) { - threads[i].join(); - } - - ArrayList bdbCookieArrayList = new ArrayList(bdbCookieStore().getCookies()); - Map domainCounts = new HashMap(); - for (Cookie cookie : bdbCookieArrayList) { - if (domainCounts.get(cookie.getDomain()) == null) { - domainCounts.put(cookie.getDomain(), 1); - } - else { - domainCounts.put(cookie.getDomain(), domainCounts.get(cookie.getDomain()) + 1); - } - } - - for (String domain: domainCounts.keySet()) { - // the cookie store intentionally doesn't synchronize so we allow up to thread.length - // additional cookies over the limit - assertTrue(domainCounts.get(domain) <= BdbCookieStore.MAX_COOKIES_FOR_DOMAIN + threads.length); - } - } - protected void assertCookieStoreCountEquals(BdbCookieStore bdb, int count) { assertEquals(bdb.getCookies().size(), count); } From 5612fa249b0d4263c948872594acfeccd792cf9d Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 27 Aug 2019 23:53:34 +0900 Subject: [PATCH 20/23] Use super.getVariants() rather than super.getVariants(GET) This was a regression introduced in the upgrade to Restlet 2. I encountered a NullPointerException here when upgrading and misunderstood the cause of it. Since PUT and DELETE return no content they are actually supposed to return null. --- .../crawler/restlet/EnhDirectoryResource.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java index 18c59354..05a0bc1b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java @@ -50,7 +50,11 @@ public class EnhDirectoryResource extends DirectoryServerResource { */ @Override public List getVariants() { - List variants = new LinkedList<>(super.getVariants(Method.GET)); + List superVariants = super.getVariants(); + if (superVariants == null) { + return null; // PUT and DELETE return no content + } + List variants = new LinkedList<>(superVariants); Form f = getRequest().getResourceRef().getQueryAsForm(); String format = f.getFirstValue("format"); if("textedit".equals(format)) { @@ -64,7 +68,11 @@ public class EnhDirectoryResource extends DirectoryServerResource { } catch (Exception e) { throw new RuntimeException(e); } - variants = new LinkedList<>(super.getVariants(Method.GET)); + superVariants = super.getVariants(); + if (superVariants == null) { + return null; + } + variants = new LinkedList<>(superVariants); } // wrap FileRepresentations in EditRepresentations ListIterator iter = variants.listIterator(); From eeddfd763f8a0199b01fa0119d24c335de128183 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 28 Aug 2019 12:46:29 +0900 Subject: [PATCH 21/23] Override PUT so it doesn't change the file extension Fixes #282 and HER-1907 --- .../crawler/restlet/EnhDirectoryResource.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java index 05a0bc1b..ffebd7e6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java @@ -21,6 +21,8 @@ package org.archive.crawler.restlet; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.util.LinkedList; @@ -153,4 +155,28 @@ public class EnhDirectoryResource extends DirectoryServerResource { getResponse().redirectSeeOther(ref); return new EmptyRepresentation(); } + + /* + * XXX: We override Restlet's default PUT behaviour (see FileClientHelper.handleFilePut) as it unhelpfully changes + * the file extension based on the content-type and there's no apparent way to disable that. + */ + @Override + public Representation put(Representation entity) throws ResourceException { + File file = new File(URI.create(getTargetUri())); + if (getTargetUri().endsWith("/") || file.isDirectory()) { + return super.put(entity); + } + boolean created = !file.exists(); + try (FileOutputStream out = new FileOutputStream(file)) { + entity.write(out); + } catch (FileNotFoundException e) { + throw new ResourceException(Status.CLIENT_ERROR_NOT_FOUND, e); + } catch (IOException e) { + throw new ResourceException(Status.SERVER_ERROR_INTERNAL, e); + } + if (created) { + getResponse().setStatus(Status.SUCCESS_CREATED); + } + return new EmptyRepresentation(); + } } From 0cee24922fea80acd0c7ef357b32168e35f2c195 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 11 Oct 2019 19:44:35 +0000 Subject: [PATCH 22/23] AssignmentLevelSurtQueueAssignmentPolicy.java - Add support for forced queue assignment and parallel queues URIAuthorityBasedQueueAssignmentPolicy.java - Add interoperability between forced queue assignment and parallel queues QuotaEnforcer.java - Fix javadoc to match default behavior --- ...ignmentLevelSurtQueueAssignmentPolicy.java | 33 +++++++++++++++++-- ...RIAuthorityBasedQueueAssignmentPolicy.java | 8 ++--- .../crawler/prefetch/QuotaEnforcer.java | 2 +- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java index d187bfc6..88603e9c 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java @@ -18,7 +18,9 @@ */ package org.archive.crawler.frontier; +import org.apache.commons.lang.StringUtils; import org.archive.modules.CrawlURI; +import org.archive.net.UURI; import org.archive.net.PublicSuffixes; /** @@ -32,9 +34,34 @@ public class AssignmentLevelSurtQueueAssignmentPolicy extends private static final long serialVersionUID = -1533545293624791702L; @Override - public String getClassKey(CrawlURI cauri) { - String candidate = super.getClassKey(cauri); - candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + public String getClassKey(CrawlURI curi) { + if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { + return curi.getClassKey(); + } + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = super.getClassKey(curi); + candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + + if(!StringUtils.isEmpty(getForceQueueAssignment())) { + candidate = getForceQueueAssignment(); + } + + // all whois urls in the same queue + if (curi.getUURI().getScheme().equals("whois")) { + return "whois..."; + } + + if(StringUtils.isEmpty(candidate)) { + return DEFAULT_CLASS_KEY; + } + if(getParallelQueues()>1) { + int subqueue = getSubqueue(basis,getParallelQueues()); + if (subqueue>0) { + candidate += "+"+subqueue; + } + } + return candidate; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 8001540d..4fcec33e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -90,9 +90,12 @@ implements if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { return curi.getClassKey(); } + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = getCoreKey(basis); if(!StringUtils.isEmpty(getForceQueueAssignment())) { - return getForceQueueAssignment(); + candidate = getForceQueueAssignment(); } // all whois urls in the same queue @@ -100,9 +103,6 @@ implements return "whois..."; } - UURI basis = curi.getPolicyBasisUURI(); - String candidate = getCoreKey(basis); - if(StringUtils.isEmpty(candidate)) { return DEFAULT_CLASS_KEY; } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index d7630227..171bb5d0 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -348,7 +348,7 @@ public class QuotaEnforcer extends Processor { * being force-retired (if the Frontier supports this). Note that if your * queues combine URIs that are different with regard to the quota category, * the retirement may hold back URIs not in the same quota category. Default - * is false. + * is true. */ { setForceRetire(true); From 74c0739bb00bf084a64a44aefea0bd6bd452098e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 11 Oct 2019 13:49:49 -0700 Subject: [PATCH 23/23] fix line ending and indentation issues --- .../AssignmentLevelSurtQueueAssignmentPolicy.java | 2 +- .../frontier/URIAuthorityBasedQueueAssignmentPolicy.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java index 88603e9c..75875184 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java @@ -41,7 +41,7 @@ public class AssignmentLevelSurtQueueAssignmentPolicy extends UURI basis = curi.getPolicyBasisUURI(); String candidate = super.getClassKey(curi); - candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); if(!StringUtils.isEmpty(getForceQueueAssignment())) { candidate = getForceQueueAssignment(); diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 4fcec33e..3212c4f0 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -90,12 +90,12 @@ implements if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { return curi.getClassKey(); } - - UURI basis = curi.getPolicyBasisUURI(); - String candidate = getCoreKey(basis); + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = getCoreKey(basis); if(!StringUtils.isEmpty(getForceQueueAssignment())) { - candidate = getForceQueueAssignment(); + candidate = getForceQueueAssignment(); } // all whois urls in the same queue