diff --git a/.classpath b/.classpath index 0f16478b..dbeb1869 100644 --- a/.classpath +++ b/.classpath @@ -9,7 +9,7 @@ - + @@ -21,11 +21,7 @@ - - - - - + @@ -33,7 +29,7 @@ - + @@ -59,6 +55,6 @@ - + diff --git a/commons/src/main/java/org/archive/net/DownloadURLConnection.java b/commons/src/main/java/org/archive/net/DownloadURLConnection.java index c98eae11..fbcee421 100644 --- a/commons/src/main/java/org/archive/net/DownloadURLConnection.java +++ b/commons/src/main/java/org/archive/net/DownloadURLConnection.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import java.util.Arrays; import java.util.logging.Level; import java.util.logging.Logger; @@ -84,7 +85,7 @@ public abstract class DownloadURLConnection extends URLConnection { } ProcessResult pr = ProcessUtils.exec(cmd); if (pr.getResult() != 0) { - LOGGER.info(cmd + " returned non-null " + pr.getResult()); + LOGGER.info(Arrays.toString(cmd) + " returned non-null " + pr.getResult()); } // Assume download went smoothly. this.connected = true; diff --git a/commons/src/main/java/org/archive/net/s3/S3URLConnection.java b/commons/src/main/java/org/archive/net/s3/S3URLConnection.java index e991766e..668dfb43 100644 --- a/commons/src/main/java/org/archive/net/s3/S3URLConnection.java +++ b/commons/src/main/java/org/archive/net/s3/S3URLConnection.java @@ -53,7 +53,7 @@ public class S3URLConnection extends URLConnection { object = service.getObject(bucket, url.getPath().substring(1)); } catch (S3ServiceException s3e) { s3e.printStackTrace(); - throw new IOException("Error connecting to S3: " + s3e.getStackTrace()); + throw new IOException("Error connecting to S3: " + s3e, s3e); } } @@ -82,7 +82,7 @@ public class S3URLConnection extends URLConnection { } catch (S3ServiceException s3e) { s3e.printStackTrace(); - throw new IOException("Error reading from S3: " + s3e.getStackTrace()); + throw new IOException("Error reading from S3: " + s3e, s3e); } } } diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java index 9805567d..07545dbf 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java @@ -55,14 +55,15 @@ implements ObjectIdentityCache { public V getOrUse(String key, Supplier supplierOrNull) { V val = map.get(key); - if (val==null && supplierOrNull!=null) { + if (val != null) { + val.setIdentityCache(this); + } else if (supplierOrNull!=null) { val = supplierOrNull.get(); V prevVal = map.putIfAbsent(key, val); if(prevVal!=null) { val = prevVal; } } - val.setIdentityCache(this); return val; } diff --git a/commons/src/main/java/org/archive/util/ProcessUtils.java b/commons/src/main/java/org/archive/util/ProcessUtils.java index d3139791..da827090 100644 --- a/commons/src/main/java/org/archive/util/ProcessUtils.java +++ b/commons/src/main/java/org/archive/util/ProcessUtils.java @@ -22,6 +22,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Arrays; import java.util.logging.Level; import java.util.logging.Logger; @@ -135,7 +136,7 @@ public class ProcessUtils { try { exitVal = p.waitFor(); } catch (InterruptedException e) { - throw new IOException("Wait on process " + args + " interrupted: " + throw new IOException("Wait on process " + Arrays.toString(args) + " interrupted: " + e.getMessage()); } ProcessUtils.ProcessResult result = diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java index ee8fdb04..3526036b 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -581,7 +581,7 @@ implements Serializable, /** * @return CrawlController state. */ - public Object getState() { + public State getState() { return this.state; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index 286e6646..3423487f 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -36,10 +36,10 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.Queue; import java.util.Set; import java.util.SortedMap; -import java.util.Map.Entry; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.DelayQueue; @@ -1289,15 +1289,16 @@ implements Closeable, q = null; } } - - if(q == null) { + + if(q != null) { + if(!legendWritten) { + writer.println(q.shortReportLegend()); + legendWritten = true; + } + q.shortReportLineTo(writer); + } else { writer.print(" ERROR: "+obj); } - if(!legendWritten) { - writer.println(q.shortReportLegend()); - legendWritten = true; - } - q.shortReportLineTo(writer); } } @@ -1459,7 +1460,7 @@ implements Closeable, * @param total * @param max */ - @SuppressWarnings("unchecked") + @SuppressWarnings("rawtypes") protected void appendQueueReports(PrintWriter w, String label, Iterator iterator, int total, int max) { Object obj; @@ -1479,11 +1480,12 @@ implements Closeable, } else { q = this.allQueues.get((String)obj); } - if(q == null) { + if(q != null) { + w.println(label+"#"+count+":"); + q.reportTo(w); + } else { w.print("WARNING: No report for queue "+obj); } - w.println(label+"#"+count+":"); - q.reportTo(w); } count++; if(count < total) { diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 64525768..39c648f3 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -191,14 +191,6 @@ public class CrawlerLoggerModule */ private transient Logger progressStats; - /** - * Logger to hold job summary report. - * - * Large state reports made at infrequent intervals (e.g. job ending) go - * here. - */ - private transient Logger reports; - /** * Record of fileHandlers established for loggers, * assisting file rotation. @@ -402,12 +394,6 @@ public class CrawlerLoggerModule return progressStats; } - - public Logger getReports() { - return reports; - } - - public Logger getRuntimeErrors() { return runtimeErrors; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java index 8420b359..c13b4838 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java @@ -34,7 +34,7 @@ public class DecideRuleSequence extends DecideRule implements BeanNameAware, Lif Logger.getLogger(DecideRuleSequence.class.getName()); private static final long serialVersionUID = 3L; - protected Logger fileLogger = null; + protected transient Logger fileLogger = null; /** * If enabled, log decisions to file named logs/{spring-bean-id}.log. Format diff --git a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java index 05511c5b..4b4b059a 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java @@ -18,6 +18,7 @@ */ package org.archive.modules.deciderules; +import java.io.Serializable; import java.net.InetAddress; /** @@ -26,7 +27,7 @@ import java.net.InetAddress; * @version $Date$, $Revision$ * @see ExternalImplDecideRule */ -public interface ExternalGeoLookupInterface { +public interface ExternalGeoLookupInterface extends Serializable { /** * @param obj Object to evaluate * @return String of a country code that the object belongs to. diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java index 556a8e79..14b64c27 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java @@ -23,7 +23,6 @@ import java.io.InputStream; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.apache.commons.io.IOUtils; import org.archive.modules.CrawlURI; import org.archive.net.UURI; @@ -50,9 +49,7 @@ import org.archive.net.UURIFactory; */ public class ExtractorUniversal extends ContentExtractor { - private static final long serialVersionUID = 3L; - /** * How deep to look into files for URI strings, in bytes. @@ -485,7 +482,7 @@ public class ExtractorUniversal extends ContentExtractor { return false; } - potentialTLD.toLowerCase(); + potentialTLD = potentialTLD.toLowerCase(); Matcher uri = TLDs.matcher(potentialTLD); boolean ret = uri.matches(); return ret; diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java index 2026c159..f1aedae2 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java @@ -75,8 +75,7 @@ public class BdbCookieStorage extends AbstractCookieStorage implements Checkpoin new StringBinding(), new SerialBinding(classCatalog,Cookie.class), true); - SortedMap result = cookies; - return result; + return cookies; } catch (DatabaseException e) { throw new RuntimeException(e); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 605eab5c..dc7242a8 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -84,11 +84,11 @@ import org.apache.commons.httpclient.protocol.Protocol; import org.apache.commons.httpclient.protocol.ProtocolSocketFactory; import org.apache.commons.lang.StringUtils; import org.archive.httpclient.ConfigurableX509TrustManager; +import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; import org.archive.httpclient.HttpRecorderGetMethod; import org.archive.httpclient.HttpRecorderMethod; import org.archive.httpclient.HttpRecorderPostMethod; import org.archive.httpclient.SingleHttpConnectionManager; -import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; import org.archive.io.RecorderLengthExceededException; import org.archive.io.RecorderTimeoutException; import org.archive.io.RecorderTooMuchHeaderException; @@ -1306,7 +1306,8 @@ public class FetchHTTP extends Processor implements Lifecycle { if (key == null || key.length() <= 0 || challenge == null || challenge.length() <= 0) { logger.warning("Empty scheme: " + curi.toString() + ": " - + headers); + + Arrays.toString(headers)); + continue; } AuthScheme authscheme = null; if (key.equals("basic")) { @@ -1321,7 +1322,7 @@ public class FetchHTTP extends Processor implements Lifecycle { try { authscheme.processChallenge(challenge); } catch (MalformedChallengeException e) { - logger.fine(e.getMessage() + " " + curi + " " + headers); + logger.fine(e.getMessage() + " " + curi + " " + Arrays.toString(headers)); continue; } if (authscheme.isConnectionBased()) { diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index ad535d78..e314c965 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -31,10 +31,9 @@ import java.util.HashSet; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.zip.Checksum; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.Predicate; +import org.apache.commons.collections.PredicateUtils; import org.apache.commons.httpclient.NoHttpResponseException; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; @@ -70,7 +69,6 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden protected Robotstxt robotstxt; long robotsFetched = ROBOTS_NOT_FETCHED; boolean validRobots = false; - Checksum robotstxtChecksum; FetchStats substats = new FetchStats(); // how many consecutive connection errors have been encountered; @@ -152,14 +150,12 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden } // special deeming for a particular kind of connection-lost (empty server response) - if(curi.getFetchStatus() == S_CONNECT_LOST && CollectionUtils.exists(curi.getNonFatalFailures(),new Predicate() { - public boolean evaluate(Object obj) { - return obj instanceof NoHttpResponseException; - } - })) { - curi.setFetchStatus(S_DEEMED_NOT_FOUND); - gotSomething = true; - } + if (curi.getFetchStatus() == S_CONNECT_LOST + && CollectionUtils.exists(curi.getNonFatalFailures(), + PredicateUtils.instanceofPredicate(NoHttpResponseException.class))) { + curi.setFetchStatus(S_DEEMED_NOT_FOUND); + gotSomething = true; + } if (!gotSomething) { // robots.txt fetch failed and exceptions (ignore/deeming) don't apply; no valid robots info yet diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java index ce99725a..a8767735 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java @@ -29,7 +29,6 @@ import org.archive.checkpointing.Checkpointable; import org.archive.io.CrawlerJournal; import org.archive.modules.CrawlURI; import org.archive.spring.ConfigPath; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.Lifecycle; @@ -106,12 +105,6 @@ implements Checkpointable, Lifecycle { public void finishCheckpoint(Checkpoint checkpointInProgress) {} - Checkpoint recoveryCheckpoint; - @Autowired(required=false) - public void setRecoveryCheckpoint(Checkpoint checkpoint) { - this.recoveryCheckpoint = checkpoint; - } - @Override protected boolean shouldProcess(CrawlURI uri) { return shouldStore(uri); diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index cdaecef5..36c62d9e 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -160,18 +160,18 @@ public class ARCWriterProcessor extends WriterPoolProcessor { setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - position)); getPool().returnFile(writer); - } - - String filename = writer.getFile().getName(); - if (filename.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) { - filename = filename.substring(0, filename.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length()); - } - curi.addExtraInfo("arcFilename", filename); - - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); - if (history != null && history[0] != null) { - history[0].put(A_WRITE_TAG, filename); + + String filename = writer.getFile().getName(); + if (filename.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) { + filename = filename.substring(0, filename.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length()); + } + curi.addExtraInfo("arcFilename", filename); + + @SuppressWarnings("unchecked") + Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + if (history != null && history[0] != null) { + history[0].put(A_WRITE_TAG, filename); + } } } return checkBytesWritten();