lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 14:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Various little cleanups including several discovered by Aaron using an automated tool.

Browse Source 
* WorkQueueFrontier.java
   (HER-1925) avoid 2 possible null pointer dereferences
* CrawlController.java
    getState() - change declared return type from Object to State
* CrawlerLoggerModule.java
    (HER-1932) remove unused, unset field "reports"
* BdbCookieStorage.java
    elide pointless extra variable
* FetchHTTP.java, DownloadURLConnection.java, ProcessUtils.java
   (HER-1933) use Arrays.toString() for logged arrays
* CrawlServer.java
    - remove unused field robotstxtChecksum
    - updateRobots() - avoid reinventing existing utility class InstanceofPredicate
* ExternalGeoLookupInterface.java
    (HER-1938) extend Serializable, since ExternalGeoLocationDecideRule is declared Serializable and has a ExternalGeoLookupInterface field
* DecideRuleSequence.java
    (HER-1937) make field fileLogger transient
* PersistLogProcessor.java
    (HER-1924) remove field recoveryCheckpoint shadowing same field in superclass Processor
* ExtractorUniversal.java
    (HER-1920) use return value of potentialTLD.toLowerCase() as it appears was intended
* ARCWriterProcessor.java
    (HER-1928) avoid possible null pointer dereference
* S3URLConnection.java
    (HER-1933) set S3ServiceException as cause of rethrown IOException, and do not put stacktrace array in message
* ObjectIdentityMemCache.java
    (HER-1929) avoid possible null pointer dereference
* .classpath
    more source jar references, other cleanup
This commit is contained in:
nlevitt
2011-09-12 02:12:44 +00:00
parent e38f0a73ba
commit 24f4b715ec
16 changed files with 56 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.classpath
+4 -8
View File
@@ -9,7 +9,7 @@
<classpathentry kind="src" path="dist/src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="var" path="M2_REPO/org/beanshell/bsh/2.0b5/bsh-2.0b5.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.14/jetty-6.1.14.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.14/jetty-6.1.14.jar" sourcepath="/M2_REPO/org/mortbay/jetty/jetty/6.1.14/jetty-6.1.14-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1.jar" sourcepath="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4.jar" sourcepath="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804.jar" sourcepath="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804-sources.jar"/>
@@ -21,11 +21,7 @@
<classpathentry kind="var" path="M2_REPO/com/anotherbigidea/javaswf/CVS-SNAPSHOT-1/javaswf-CVS-SNAPSHOT-1.jar"/>
<classpathentry kind="var" path="M2_REPO/it/unimi/dsi/mg4j/1.0.1/mg4j-1.0.1.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3.jar" sourcepath="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/fastutil/fastutil/5.0.7/fastutil-5.0.7.jar">
<attributes>
<attribute name="javadoc_location" value="jar:file://Users/pjack/.m2/repository/fastutil/fastutil/5.0.7/fastutil-5.0.7-javadoc.jar!/"/>
</attributes>
</classpathentry>
<classpathentry kind="var" path="M2_REPO/fastutil/fastutil/5.0.7/fastutil-5.0.7.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.5.0/jets3t-0.5.0.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.14/jetty-util-6.1.14.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-net/commons-net/2.0/commons-net-2.0.jar" sourcepath="/M2_REPO/commons-net/commons-net/2.0/commons-net-2.0-sources.jar"/>
@@ -33,7 +29,7 @@
<classpathentry kind="var" path="M2_REPO/poi/poi-scratchpad/2.5.1-final-20040804/poi-scratchpad-2.5.1-final-20040804.jar"/>
<classpathentry kind="var" path="M2_REPO/com/sleepycat/je/4.1.6/je-4.1.6.jar" sourcepath="/M2_REPO/com/sleepycat/je/4.1.6/je-4.1.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-io/commons-io/1.4/commons-io-1.4.jar" sourcepath="/M2_REPO/commons-io/commons-io/1.4/commons-io-1.4-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/archive/overlays/archive-overlay-commons-httpclient/3.1/archive-overlay-commons-httpclient-3.1.jar" sourcepath="/M2SRC/commons-httpclient-3.1/src/java"/>
<classpathentry kind="var" path="M2_REPO/org/archive/overlays/archive-overlay-commons-httpclient/3.1/archive-overlay-commons-httpclient-3.1.jar" sourcepath="/M2_REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/dnsjava/dnsjava/2.0.3/dnsjava-2.0.3.jar" sourcepath="/M2SRC/dnsjava-2.0.3"/>
<classpathentry kind="var" path="M2_REPO/org/springframework/spring-core/3.0.5.RELEASE/spring-core-3.0.5.RELEASE.jar" sourcepath="/M2_REPO/org/springframework/spring-core/3.0.5.RELEASE/spring-core-3.0.5.RELEASE-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/springframework/spring-beans/3.0.5.RELEASE/spring-beans-3.0.5.RELEASE.jar" sourcepath="/M2_REPO/org/springframework/spring-beans/3.0.5.RELEASE/spring-beans-3.0.5.RELEASE-sources.jar"/>
@@ -59,6 +55,6 @@
<classpathentry kind="var" path="M2_REPO/com/esotericsoftware/reflectasm/0.8/reflectasm-0.8.jar"/>
<classpathentry kind="var" path="M2_REPO/asm/asm/3.2/asm-3.2.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/guava/guava/r08/guava-r08.jar" sourcepath="/M2_REPO/com/google/guava/guava/r08/guava-r08-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3.jar" sourcepath="/M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3-sources.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
commons/src/main/java/org/archive/net/DownloadURLConnection.java
+2 -1
View File
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Arrays;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -84,7 +85,7 @@ public abstract class DownloadURLConnection extends URLConnection {
}
ProcessResult pr = ProcessUtils.exec(cmd);
if (pr.getResult() != 0) {
LOGGER.info(cmd + " returned non-null " + pr.getResult());
LOGGER.info(Arrays.toString(cmd) + " returned non-null " + pr.getResult());
}
// Assume download went smoothly.
this.connected = true;
commons/src/main/java/org/archive/net/s3/S3URLConnection.java
+2 -2
View File
@@ -53,7 +53,7 @@ public class S3URLConnection extends URLConnection {
object = service.getObject(bucket, url.getPath().substring(1));
} catch (S3ServiceException s3e) {
s3e.printStackTrace();
throw new IOException("Error connecting to S3: " + s3e.getStackTrace());
throw new IOException("Error connecting to S3: " + s3e, s3e);
}
}
@@ -82,7 +82,7 @@ public class S3URLConnection extends URLConnection {
}
catch (S3ServiceException s3e) {
s3e.printStackTrace();
throw new IOException("Error reading from S3: " + s3e.getStackTrace());
throw new IOException("Error reading from S3: " + s3e, s3e);
}
}
}
commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java
+3 -2
View File
@@ -55,14 +55,15 @@ implements ObjectIdentityCache<V> {
public V getOrUse(String key, Supplier<V> supplierOrNull) {
V val = map.get(key);
if (val==null && supplierOrNull!=null) {
if (val != null) {
val.setIdentityCache(this);
} else if (supplierOrNull!=null) {
val = supplierOrNull.get();
V prevVal = map.putIfAbsent(key, val);
if(prevVal!=null) {
val = prevVal;
}
}
val.setIdentityCache(this);
return val;
}
commons/src/main/java/org/archive/util/ProcessUtils.java
+2 -1
View File
@@ -22,6 +22,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -135,7 +136,7 @@ public class ProcessUtils {
try {
exitVal = p.waitFor();
} catch (InterruptedException e) {
throw new IOException("Wait on process " + args + " interrupted: "
throw new IOException("Wait on process " + Arrays.toString(args) + " interrupted: "
+ e.getMessage());
}
ProcessUtils.ProcessResult result =
engine/src/main/java/org/archive/crawler/framework/CrawlController.java
+1 -1
View File
@@ -581,7 +581,7 @@ implements Serializable,
/**
* @return CrawlController state.
*/
public Object getState() {
public State getState() {
return this.state;
}
engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+14 -12
View File
@@ -36,10 +36,10 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
import java.util.SortedMap;
import java.util.Map.Entry;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue;
@@ -1289,15 +1289,16 @@ implements Closeable,
q = null;
}
}
if(q == null) {
if(q != null) {
if(!legendWritten) {
writer.println(q.shortReportLegend());
legendWritten = true;
}
q.shortReportLineTo(writer);
} else {
writer.print(" ERROR: "+obj);
}
if(!legendWritten) {
writer.println(q.shortReportLegend());
legendWritten = true;
}
q.shortReportLineTo(writer);
}
}
@@ -1459,7 +1460,7 @@ implements Closeable,
* @param total
* @param max
*/
@SuppressWarnings("unchecked")
@SuppressWarnings("rawtypes")
protected void appendQueueReports(PrintWriter w, String label, Iterator<?> iterator,
int total, int max) {
Object obj;
@@ -1479,11 +1480,12 @@ implements Closeable,
} else {
q = this.allQueues.get((String)obj);
}
if(q == null) {
if(q != null) {
w.println(label+"#"+count+":");
q.reportTo(w);
} else {
w.print("WARNING: No report for queue "+obj);
}
w.println(label+"#"+count+":");
q.reportTo(w);
}
count++;
if(count < total) {
engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java
-14
View File
@@ -191,14 +191,6 @@ public class CrawlerLoggerModule
*/
private transient Logger progressStats;
/**
* Logger to hold job summary report.
*
* Large state reports made at infrequent intervals (e.g. job ending) go
* here.
*/
private transient Logger reports;
/**
* Record of fileHandlers established for loggers,
* assisting file rotation.
@@ -402,12 +394,6 @@ public class CrawlerLoggerModule
return progressStats;
}
public Logger getReports() {
return reports;
}
public Logger getRuntimeErrors() {
return runtimeErrors;
}
modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java
+1 -1
View File
@@ -34,7 +34,7 @@ public class DecideRuleSequence extends DecideRule implements BeanNameAware, Lif
Logger.getLogger(DecideRuleSequence.class.getName());
private static final long serialVersionUID = 3L;
protected Logger fileLogger = null;
protected transient Logger fileLogger = null;
/**
* If enabled, log decisions to file named logs/{spring-bean-id}.log. Format
modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java
+2 -1
View File
@@ -18,6 +18,7 @@
*/
package org.archive.modules.deciderules;
import java.io.Serializable;
import java.net.InetAddress;
/**
@@ -26,7 +27,7 @@ import java.net.InetAddress;
* @version $Date$, $Revision$
* @see ExternalImplDecideRule
*/
public interface ExternalGeoLookupInterface {
public interface ExternalGeoLookupInterface extends Serializable {
/**
* @param obj Object to evaluate
* @return String of a country code that the object belongs to.
modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java
+1 -4
View File
@@ -23,7 +23,6 @@ import java.io.InputStream;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
@@ -50,9 +49,7 @@ import org.archive.net.UURIFactory;
*/
public class ExtractorUniversal extends ContentExtractor {
private static final long serialVersionUID = 3L;
/**
* How deep to look into files for URI strings, in bytes.
@@ -485,7 +482,7 @@ public class ExtractorUniversal extends ContentExtractor {
return false;
}
potentialTLD.toLowerCase();
potentialTLD = potentialTLD.toLowerCase();
Matcher uri = TLDs.matcher(potentialTLD);
boolean ret = uri.matches();
return ret;
modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java
+1 -2
View File
@@ -75,8 +75,7 @@ public class BdbCookieStorage extends AbstractCookieStorage implements Checkpoin
new StringBinding(),
new SerialBinding<Cookie>(classCatalog,Cookie.class),
true);
SortedMap<String,Cookie> result = cookies;
return result;
return cookies;
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+4 -3
View File
@@ -84,11 +84,11 @@ import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.apache.commons.lang.StringUtils;
import org.archive.httpclient.ConfigurableX509TrustManager;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.httpclient.HttpRecorderGetMethod;
import org.archive.httpclient.HttpRecorderMethod;
import org.archive.httpclient.HttpRecorderPostMethod;
import org.archive.httpclient.SingleHttpConnectionManager;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.io.RecorderLengthExceededException;
import org.archive.io.RecorderTimeoutException;
import org.archive.io.RecorderTooMuchHeaderException;
@@ -1306,7 +1306,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
if (key == null || key.length() <= 0 || challenge == null
|| challenge.length() <= 0) {
logger.warning("Empty scheme: " + curi.toString() + ": "
+ headers);
+ Arrays.toString(headers));
continue;
}
AuthScheme authscheme = null;
if (key.equals("basic")) {
@@ -1321,7 +1322,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
try {
authscheme.processChallenge(challenge);
} catch (MalformedChallengeException e) {
logger.fine(e.getMessage() + " " + curi + " " + headers);
logger.fine(e.getMessage() + " " + curi + " " + Arrays.toString(headers));
continue;
}
if (authscheme.isConnectionBased()) {
modules/src/main/java/org/archive/modules/net/CrawlServer.java
+7 -11
View File
@@ -31,10 +31,9 @@ import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.Checksum;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
@@ -70,7 +69,6 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden
protected Robotstxt robotstxt;
long robotsFetched = ROBOTS_NOT_FETCHED;
boolean validRobots = false;
Checksum robotstxtChecksum;
FetchStats substats = new FetchStats();
// how many consecutive connection errors have been encountered;
@@ -152,14 +150,12 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden
}
// special deeming for a particular kind of connection-lost (empty server response)
if(curi.getFetchStatus() == S_CONNECT_LOST && CollectionUtils.exists(curi.getNonFatalFailures(),new Predicate() {
public boolean evaluate(Object obj) {
return obj instanceof NoHttpResponseException;
}
})) {
curi.setFetchStatus(S_DEEMED_NOT_FOUND);
gotSomething = true;
}
if (curi.getFetchStatus() == S_CONNECT_LOST
&& CollectionUtils.exists(curi.getNonFatalFailures(),
PredicateUtils.instanceofPredicate(NoHttpResponseException.class))) {
curi.setFetchStatus(S_DEEMED_NOT_FOUND);
gotSomething = true;
}
if (!gotSomething) {
// robots.txt fetch failed and exceptions (ignore/deeming) don't apply; no valid robots info yet
modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java
-7
View File
@@ -29,7 +29,6 @@ import org.archive.checkpointing.Checkpointable;
import org.archive.io.CrawlerJournal;
import org.archive.modules.CrawlURI;
import org.archive.spring.ConfigPath;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.Lifecycle;
@@ -106,12 +105,6 @@ implements Checkpointable, Lifecycle {
public void finishCheckpoint(Checkpoint checkpointInProgress) {}
Checkpoint recoveryCheckpoint;
@Autowired(required=false)
public void setRecoveryCheckpoint(Checkpoint checkpoint) {
this.recoveryCheckpoint = checkpoint;
}
@Override
protected boolean shouldProcess(CrawlURI uri) {
return shouldStore(uri);
modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java
+12 -12
View File
@@ -160,18 +160,18 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
setTotalBytesWritten(getTotalBytesWritten() +
(writer.getPosition() - position));
getPool().returnFile(writer);
}
String filename = writer.getFile().getName();
if (filename.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) {
filename = filename.substring(0, filename.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length());
}
curi.addExtraInfo("arcFilename", filename);
@SuppressWarnings("unchecked")
Map<String,Object>[] history = (Map<String,Object>[])curi.getData().get(A_FETCH_HISTORY);
if (history != null && history[0] != null) {
history[0].put(A_WRITE_TAG, filename);
String filename = writer.getFile().getName();
if (filename.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) {
filename = filename.substring(0, filename.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length());
}
curi.addExtraInfo("arcFilename", filename);
@SuppressWarnings("unchecked")
Map<String,Object>[] history = (Map<String,Object>[])curi.getData().get(A_FETCH_HISTORY);
if (history != null && history[0] != null) {
history[0].put(A_WRITE_TAG, filename);
}
}
}
return checkBytesWritten();