From 4b047dba6453ebb964e7300e6f3f1403608d5ff0 Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Tue, 3 Jul 2012 13:13:50 -0700 Subject: [PATCH 1/6] count path segments by looking at the path * TooManyPathSegmentsDecideRule evaluate(CrawlURI) previously counted slashes in the full URI. Now it only counts slashes in the path of the URI, meaning a URI with slashes in the query string will not be counted as having more path segments than it has in its path. --- .../modules/deciderules/TooManyPathSegmentsDecideRule.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java index 85268b2d..4ec9e7bb 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java @@ -63,11 +63,11 @@ public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { */ @Override protected boolean evaluate(CrawlURI curi) { - String uri = curi.toString(); + String uriPath = curi.getUURI().getEscapedPath(); int count = 0; int threshold = getMaxPathDepth(); - for (int i = 0; i < uri.length(); i++) { - if (uri.charAt(i) == '/') { + for (int i = 0; i < uriPath.length(); i++) { + if (uriPath.charAt(i) == '/') { count++; } if (count > threshold) { From 4ca6e9cc997e5a7e942c450903d694e532ff4ce0 Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Tue, 3 Jul 2012 17:38:43 -0700 Subject: [PATCH 2/6] revert a change that may break bdb history --- .../main/java/org/apache/commons/httpclient/Cookie.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/commons/src/main/java/org/apache/commons/httpclient/Cookie.java b/commons/src/main/java/org/apache/commons/httpclient/Cookie.java index 1db473a8..5f065098 100644 --- a/commons/src/main/java/org/apache/commons/httpclient/Cookie.java +++ b/commons/src/main/java/org/apache/commons/httpclient/Cookie.java @@ -61,8 +61,8 @@ import org.apache.commons.logging.LogFactory; * * @version $Revision$ $Date$ */ -@SuppressWarnings({"serial"}) // <- HERITRIX CHANGE -public class Cookie extends NameValuePair implements Serializable, Comparator { +@SuppressWarnings({"serial","unchecked"}) // <- HERITRIX CHANGE +public class Cookie extends NameValuePair implements Serializable, Comparator { // ----------------------------------------------------------- Constructors @@ -415,7 +415,6 @@ public class Cookie extends NameValuePair implements Serializable, Comparator Date: Tue, 10 Jul 2012 14:12:33 -0700 Subject: [PATCH 3/6] suppress unused warnings for serialVersionUid Toward the goal of having Eclipse warnings be useful to look at, suppressing unused warnings for serialVersionUid will significantly reduce clutter. --- commons/src/main/java/org/archive/spring/Sheet.java | 1 + engine/src/main/java/org/archive/crawler/framework/Engine.java | 1 + .../java/org/archive/crawler/frontier/AbstractFrontier.java | 1 + .../main/java/org/archive/crawler/frontier/BdbFrontier.java | 1 + .../org/archive/crawler/frontier/BdbMultipleWorkQueues.java | 3 ++- .../java/org/archive/crawler/frontier/WorkQueueFrontier.java | 1 + .../org/archive/crawler/postprocessor/CandidatesProcessor.java | 1 + .../archive/crawler/postprocessor/DispositionProcessor.java | 1 + .../java/org/archive/crawler/postprocessor/LinksScoper.java | 1 + .../archive/crawler/postprocessor/LowDiskPauseProcessor.java | 1 + .../crawler/postprocessor/SupplementaryLinksScoper.java | 1 + .../java/org/archive/crawler/prefetch/CandidateScoper.java | 1 + .../java/org/archive/crawler/prefetch/FrontierPreparer.java | 1 + .../org/archive/crawler/prefetch/PreconditionEnforcer.java | 1 + .../main/java/org/archive/crawler/prefetch/Preselector.java | 1 + .../main/java/org/archive/crawler/prefetch/QuotaEnforcer.java | 1 + .../org/archive/crawler/prefetch/RuntimeLimitEnforcer.java | 1 + .../java/org/archive/crawler/processor/HashCrawlMapper.java | 1 + .../java/org/archive/crawler/processor/LexicalCrawlMapper.java | 1 + .../org/archive/crawler/reporting/CrawlerLoggerModule.java | 1 + .../java/org/archive/crawler/reporting/StatisticsTracker.java | 1 + .../java/org/archive/crawler/selftest/KeyWordProcessor.java | 1 + .../src/main/java/org/archive/modules/ScriptedProcessor.java | 1 + .../org/archive/modules/extractor/AggressiveExtractorHTML.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorCSS.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorDOC.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorHTML.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorHTTP.java | 1 + .../org/archive/modules/extractor/ExtractorImpliedURI.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorJS.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorPDF.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorSWF.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorURI.java | 1 + .../java/org/archive/modules/extractor/ExtractorUniversal.java | 1 + .../main/java/org/archive/modules/extractor/ExtractorXML.java | 1 + .../java/org/archive/modules/extractor/HTTPContentDigest.java | 1 + .../org/archive/modules/extractor/JerichoExtractorHTML.java | 1 + .../org/archive/modules/extractor/TrapSuppressExtractor.java | 1 + .../java/org/archive/modules/fetcher/BdbCookieStorage.java | 1 + .../src/main/java/org/archive/modules/fetcher/FetchDNS.java | 1 + .../src/main/java/org/archive/modules/fetcher/FetchFTP.java | 1 + .../src/main/java/org/archive/modules/fetcher/FetchHTTP.java | 1 + .../src/main/java/org/archive/modules/fetcher/FetchWhois.java | 1 + .../java/org/archive/modules/fetcher/SimpleCookieStorage.java | 1 + .../org/archive/modules/recrawl/FetchHistoryProcessor.java | 1 + .../java/org/archive/modules/recrawl/PersistLoadProcessor.java | 1 + .../java/org/archive/modules/recrawl/PersistLogProcessor.java | 1 + .../org/archive/modules/recrawl/PersistOnlineProcessor.java | 1 + .../java/org/archive/modules/recrawl/PersistProcessor.java | 1 + .../org/archive/modules/recrawl/PersistStoreProcessor.java | 1 + .../java/org/archive/modules/writer/ARCWriterProcessor.java | 1 + .../java/org/archive/modules/writer/Kw3WriterProcessor.java | 1 + .../java/org/archive/modules/writer/MirrorWriterProcessor.java | 1 + .../java/org/archive/modules/writer/WARCWriterProcessor.java | 1 + .../java/org/archive/modules/writer/WriterPoolProcessor.java | 1 + .../org/archive/modules/extractor/UnitTestUriLoggerModule.java | 1 + 56 files changed, 57 insertions(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/spring/Sheet.java b/commons/src/main/java/org/archive/spring/Sheet.java index 1153fec7..d4b7aff4 100644 --- a/commons/src/main/java/org/archive/spring/Sheet.java +++ b/commons/src/main/java/org/archive/spring/Sheet.java @@ -48,6 +48,7 @@ import org.springframework.beans.factory.annotation.Required; * */ public class Sheet implements BeanFactoryAware, BeanNameAware { + @SuppressWarnings("unused") private static final long serialVersionUID = 9129011082185864377L; /** diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java index 7d94f5aa..00ef5322 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Engine.java +++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java @@ -43,6 +43,7 @@ import org.archive.util.ArchiveUtils; * @contributor gojomo */ public class Engine { + @SuppressWarnings("unused") private static final long serialVersionUID = 4L; final public static String LOGS_DIR_NAME = "logs subdirectory"; diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 3b1cba65..95d0b632 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -89,6 +89,7 @@ public abstract class AbstractFrontier ExtractorParameters, CrawlUriReceiver, ApplicationListener { + @SuppressWarnings("unused") private static final long serialVersionUID = 555881755284996860L; private static final Logger logger = Logger .getLogger(AbstractFrontier.class.getName()); diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index e72fbf5b..cc965186 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -60,6 +60,7 @@ import com.sleepycat.je.DatabaseException; */ public class BdbFrontier extends WorkQueueFrontier implements Checkpointable, BeanNameAware { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; private static final Logger logger = diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index e38b525d..b94e58d6 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -59,7 +59,8 @@ import com.sleepycat.util.RuntimeExceptionWrapper; * @author gojomo */ public class BdbMultipleWorkQueues { - private static final long serialVersionUID = 1L; + @SuppressWarnings("unused") + private static final long serialVersionUID = 1L; private static final Logger LOGGER = Logger.getLogger(BdbMultipleWorkQueues.class.getName()); diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index f6ac6773..7b50727a 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -83,6 +83,7 @@ import com.sleepycat.je.DatabaseException; public abstract class WorkQueueFrontier extends AbstractFrontier implements Closeable, ApplicationContextAware { + @SuppressWarnings("unused") private static final long serialVersionUID = 570384305871965843L; /** diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java index 09f0ece9..c4d80f91 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java @@ -47,6 +47,7 @@ import org.springframework.beans.factory.annotation.Autowired; */ public class CandidatesProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = -3L; /** diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 0cba02c6..2735564f 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -54,6 +54,7 @@ import org.springframework.beans.factory.annotation.Autowired; * @version $Date$, $Revision$ */ public class DispositionProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = -1072728147960180091L; private static final Logger logger = Logger.getLogger(DispositionProcessor.class.getName()); diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java index c8bcc1d4..7db90061 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java @@ -54,6 +54,7 @@ import org.archive.modules.extractor.Link; */ public class LinksScoper extends Scoper { + @SuppressWarnings("unused") private static final long serialVersionUID = -3L; private static Logger LOGGER = diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java index 4143a581..113b9c6b 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java @@ -45,6 +45,7 @@ import org.springframework.beans.factory.annotation.Autowired; @Deprecated public class LowDiskPauseProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; /** diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java index 03608d18..bd78cf78 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java @@ -43,6 +43,7 @@ import org.archive.modules.deciderules.DecideRule; */ public class SupplementaryLinksScoper extends Scoper { + @SuppressWarnings("unused") private static final long serialVersionUID = -3L; private static Logger LOGGER = diff --git a/engine/src/main/java/org/archive/crawler/prefetch/CandidateScoper.java b/engine/src/main/java/org/archive/crawler/prefetch/CandidateScoper.java index ca120e02..2cec935d 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/CandidateScoper.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/CandidateScoper.java @@ -32,6 +32,7 @@ import org.archive.modules.ProcessResult; * @contributor gojomo */ public class CandidateScoper extends Scoper { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; @Override diff --git a/engine/src/main/java/org/archive/crawler/prefetch/FrontierPreparer.java b/engine/src/main/java/org/archive/crawler/prefetch/FrontierPreparer.java index 4fb26e24..7f8309dc 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/FrontierPreparer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/FrontierPreparer.java @@ -48,6 +48,7 @@ import org.springframework.beans.factory.annotation.Autowired; * @contributor gojomo */ public class FrontierPreparer extends Scoper { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; /** diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 8df658c0..0f83a739 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -51,6 +51,7 @@ import org.springframework.beans.factory.annotation.Autowired; * @author gojomo */ public class PreconditionEnforcer extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static final Logger logger = Logger.getLogger(PreconditionEnforcer.class.getName()); diff --git a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java index f687d9ae..d159d6c6 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java @@ -40,6 +40,7 @@ import org.archive.util.TextUtils; * */ public class Preselector extends Scoper { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; /** diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index e403c5d1..9fd434e8 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -42,6 +42,7 @@ import org.springframework.beans.factory.annotation.Autowired; * @version $Date$, $Revision$ */ public class QuotaEnforcer extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static final Logger LOGGER = diff --git a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java index ff5c307a..30a3298a 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java @@ -62,6 +62,7 @@ import org.springframework.beans.factory.annotation.Autowired; */ public class RuntimeLimitEnforcer extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; protected static Logger logger = Logger.getLogger( diff --git a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java index 22016b9c..40051bcf 100644 --- a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java @@ -37,6 +37,7 @@ import st.ata.util.FPGenerator; */ public class HashCrawlMapper extends CrawlMapper { + @SuppressWarnings("unused") private static final long serialVersionUID = 2L; protected Frontier frontier; diff --git a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java index 96358ae7..13b158dd 100644 --- a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java @@ -78,6 +78,7 @@ import org.springframework.beans.factory.annotation.Autowired; * @version $Date$, $Revision$ */ public class LexicalCrawlMapper extends CrawlMapper { + @SuppressWarnings("unused") private static final long serialVersionUID = 2L; /** diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 0f388199..a3d0da54 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -62,6 +62,7 @@ public class CrawlerLoggerModule implements UriErrorLoggerModule, Lifecycle, InitializingBean, Checkpointable, SimpleFileLoggerProvider, DisposableBean { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"${launchId}/logs"); diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index c7b4e376..b2144178 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -138,6 +138,7 @@ public class StatisticsTracker Runnable, Checkpointable, BeanNameAware { + @SuppressWarnings("unused") private static final long serialVersionUID = 5L; protected SeedModule seeds; diff --git a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java index 39dc26bd..56e92e17 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java +++ b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java @@ -57,6 +57,7 @@ import org.archive.modules.Processor; * @author pjack */ public class KeyWordProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; /** * Regular expression used to detect the presence of a keyword. diff --git a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java index ac983dfe..0143d8a1 100644 --- a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java +++ b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java @@ -59,6 +59,7 @@ import org.springframework.context.ApplicationContextAware; public class ScriptedProcessor extends Processor implements ApplicationContextAware, InitializingBean { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static final Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java index 348e1c5b..70f64f1e 100644 --- a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java @@ -32,6 +32,7 @@ import org.archive.modules.CrawlURI; */ public class AggressiveExtractorHTML extends ExtractorHTML { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; protected static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 85dfc543..60d13e2f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -57,6 +57,7 @@ import org.archive.util.TextUtils; public class ExtractorCSS extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 2L; private static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index 6e5a3321..dc61f77d 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -42,6 +42,7 @@ import org.archive.util.ms.Doc; */ public class ExtractorDOC extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Pattern PATTERN = Pattern.compile("HYPERLINK.*?\"(.*?)\""); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 5bbe0675..8722882d 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -61,6 +61,7 @@ import org.springframework.beans.factory.annotation.Autowired; */ public class ExtractorHTML extends ContentExtractor implements InitializingBean { + @SuppressWarnings("unused") private static final long serialVersionUID = 2L; private static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 714ff763..9ad25453 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -34,6 +34,7 @@ import org.archive.net.UURIFactory; */ public class ExtractorHTTP extends Extractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; public ExtractorHTTP() { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java index f8dc2777..b64cedb4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java @@ -51,6 +51,7 @@ import org.archive.net.UURIFactory; public class ExtractorImpliedURI extends Extractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Logger LOGGER = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 133b152e..275a9631 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -56,6 +56,7 @@ import org.archive.util.UriUtils; */ public class ExtractorJS extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 2L; private static Logger LOGGER = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java index e6ebeb97..d15e2240 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java @@ -38,6 +38,7 @@ import org.archive.util.FileUtils; */ public class ExtractorPDF extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static final Logger LOGGER = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java index fd2d31ce..8062b008 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java @@ -46,6 +46,7 @@ import com.anotherbigidea.io.InStream; */ public class ExtractorSWF extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java index 3b3a0529..54185401 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java @@ -48,6 +48,7 @@ import org.archive.util.TextUtils; public class ExtractorURI extends Extractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Logger LOGGER = diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java index 110f2cc1..9b8cbba3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java @@ -49,6 +49,7 @@ import org.archive.net.UURIFactory; */ public class ExtractorUniversal extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; /** diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java index 9700d0c3..903abfbd 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java @@ -49,6 +49,7 @@ import org.archive.util.UriUtils; */ public class ExtractorXML extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java index 10ef5785..1e23108a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java @@ -65,6 +65,7 @@ import org.archive.util.TextUtils; */ public class HTTPContentDigest extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index 5b782da3..eb1f6e42 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -76,6 +76,7 @@ import au.id.jericho.lib.html.StartTagType; @SuppressWarnings("unchecked") public class JerichoExtractorHTML extends ExtractorHTML { + @SuppressWarnings("unused") private static final long serialVersionUID = 1684681316546343615L; final private static Logger logger = diff --git a/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java b/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java index 1b47423a..c327232c 100644 --- a/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java @@ -28,6 +28,7 @@ import org.archive.modules.CrawlURI; * */ public class TrapSuppressExtractor extends ContentExtractor { + @SuppressWarnings("unused") private static final long serialVersionUID = -1028783453022579530L; /** ALIst attribute key for carrying-forward content-digest from 'via'*/ diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java index de91f5f8..da078b2c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java @@ -43,6 +43,7 @@ import com.sleepycat.je.DatabaseException; * @author pjack */ public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; protected BdbModule bdb; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 7acd541a..1c6dcac2 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -61,6 +61,7 @@ import org.xbill.DNS.Type; */ public class FetchDNS extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static Logger logger = Logger.getLogger(FetchDNS.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java index 77275cd6..0cc1742d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java @@ -62,6 +62,7 @@ import org.archive.util.Recorder; * */ public class FetchFTP extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; /** Logger for this class. */ diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index a92fe1ea..f0580160 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -121,6 +121,7 @@ import org.springframework.context.Lifecycle; * @version $Id$ */ public class FetchHTTP extends Processor implements Lifecycle { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; private static Logger logger = Logger.getLogger(FetchHTTP.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java index 0b35e31f..77c89889 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java @@ -122,6 +122,7 @@ import com.sleepycat.je.DatabaseException; public class FetchWhois extends Processor implements CoreAttributeConstants, FetchStatusCodes, Lifecycle { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; private static Logger logger = Logger.getLogger(FetchWhois.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java index c5e914ac..d815a375 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java @@ -26,6 +26,7 @@ import org.apache.commons.httpclient.Cookie; public class SimpleCookieStorage extends AbstractCookieStorage { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; final private SortedMap map = new TreeMap(); diff --git a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java index 24bb5074..ccd3a8af 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java @@ -36,6 +36,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME; * @version $Date: 2006-09-25 20:19:54 +0000 (Mon, 25 Sep 2006) $, $Revision: 4654 $ */ public class FetchHistoryProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; /** Desired history array length. */ diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java index 2dc2f8c4..345158ee 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java @@ -39,6 +39,7 @@ import com.sleepycat.je.DatabaseException; * @version $Date: 2006-09-25 20:19:54 +0000 (Mon, 25 Sep 2006) $, $Revision: 4654 $ */ public class PersistLoadProcessor extends PersistOnlineProcessor { + @SuppressWarnings("unused") private static final long serialVersionUID = -1917169316015093131L; private static final Logger logger = Logger.getLogger(PersistLoadProcessor.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java index 3e8a3a6f..d03cb5af 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java @@ -43,6 +43,7 @@ import org.springframework.context.Lifecycle; public class PersistLogProcessor extends PersistProcessor implements Checkpointable, Lifecycle { + @SuppressWarnings("unused") private static final long serialVersionUID = 1678691994065439346L; protected CrawlerJournal log; diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java index 6ced89a4..55354c5c 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java @@ -40,6 +40,7 @@ import com.sleepycat.je.DatabaseException; public abstract class PersistOnlineProcessor extends PersistProcessor implements Lifecycle { + @SuppressWarnings("unused") private static final long serialVersionUID = -666479480942267268L; protected BdbModule bdb; diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java index c40b4b2d..be0fd555 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java @@ -64,6 +64,7 @@ import com.sleepycat.je.EnvironmentConfig; */ public abstract class PersistProcessor extends AbstractPersistProcessor { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; private static final Logger logger = diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java index 8977e344..9bb7da92 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java @@ -29,6 +29,7 @@ import org.archive.modules.CrawlURI; */ public class PersistStoreProcessor extends PersistOnlineProcessor { + @SuppressWarnings("unused") private static final long serialVersionUID = -8308356194337303758L; // class description: "PersistStoreProcessor. Stores CrawlURI attributes " + diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index 36c62d9e..d0beea34 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -61,6 +61,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { final static private String METADATA_TEMPLATE = readMetadataTemplate(); + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static final Logger logger = diff --git a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java index efedbbaa..95210a7c 100644 --- a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java @@ -78,6 +78,7 @@ import org.springframework.beans.factory.annotation.Autowired; public class Kw3WriterProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static String COLON = ":"; diff --git a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java index 5080fad1..a7ea464c 100644 --- a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java @@ -86,6 +86,7 @@ import org.archive.util.FileUtils; @author Howard Lee Gayle */ public class MirrorWriterProcessor extends Processor { + @SuppressWarnings("unused") private static final long serialVersionUID = 3L; private static final Logger logger = Logger.getLogger(MirrorWriterProcessor.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index c29a16b6..7aa407f3 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -99,6 +99,7 @@ import org.archive.util.anvl.ANVLRecord; * @contributor stack */ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWriterPoolSettings { + @SuppressWarnings("unused") private static final long serialVersionUID = 6182850087635847443L; private static final Logger logger = Logger.getLogger(WARCWriterProcessor.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index dcd5d4fc..eb7a7d5a 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -61,6 +61,7 @@ import org.springframework.context.Lifecycle; */ public abstract class WriterPoolProcessor extends Processor implements Lifecycle, Checkpointable, WriterPoolSettings { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(WriterPoolProcessor.class.getName()); diff --git a/modules/src/test/java/org/archive/modules/extractor/UnitTestUriLoggerModule.java b/modules/src/test/java/org/archive/modules/extractor/UnitTestUriLoggerModule.java index bf2f03b3..a9ebd9db 100644 --- a/modules/src/test/java/org/archive/modules/extractor/UnitTestUriLoggerModule.java +++ b/modules/src/test/java/org/archive/modules/extractor/UnitTestUriLoggerModule.java @@ -14,6 +14,7 @@ import org.archive.net.UURI; * */ public class UnitTestUriLoggerModule implements UriErrorLoggerModule { + @SuppressWarnings("unused") private static final long serialVersionUID = 1L; final private static Logger LOGGER = From 9505edc9c3035bdbd78e6307c0b34fc5ad247c8c Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Tue, 10 Jul 2012 18:01:53 -0700 Subject: [PATCH 4/6] fix for null paths * TooManyPathSegmentsDecideRule Sometimes the path is null, throwing a NPE when the loop in evaluate(CrawlURI) is entered. In such a case, revert to previous behavior. --- .../deciderules/TooManyPathSegmentsDecideRule.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java index 4ec9e7bb..26c83573 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java @@ -18,6 +18,9 @@ */ package org.archive.modules.deciderules; +import java.util.logging.Level; +import java.util.logging.Logger; + import org.archive.modules.CrawlURI; /** @@ -30,6 +33,7 @@ import org.archive.modules.CrawlURI; public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { private static final long serialVersionUID = 3L; + private static final Logger logger = Logger.getLogger(TooManyPathSegmentsDecideRule.class.getName()); /** default for this class is to REJECT */ { @@ -64,6 +68,11 @@ public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { @Override protected boolean evaluate(CrawlURI curi) { String uriPath = curi.getUURI().getEscapedPath(); + if (uriPath == null) { + uriPath = curi.toString(); + // TODO this is a quick fix. figure out why getUURI().getEscapedPath() returns null. + logger.log(Level.WARNING, "getUURI().getEscapedPath() returns null for the given CrawlURI: "+ uriPath); + } int count = 0; int threshold = getMaxPathDepth(); for (int i = 0; i < uriPath.length(); i++) { From 787f7d9234a6dd467f8e34ee60ad5cdd1a15103d Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Wed, 11 Jul 2012 13:40:01 -0700 Subject: [PATCH 5/6] If the URI doesn't have a path, don't count segments. * TooManyPathSegmentsDecideRule#evaluate(CrawlURI) return false if a path can't be obtained from the uri --- .../modules/deciderules/TooManyPathSegmentsDecideRule.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java index 26c83573..3a58382f 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java @@ -18,7 +18,6 @@ */ package org.archive.modules.deciderules; -import java.util.logging.Level; import java.util.logging.Logger; import org.archive.modules.CrawlURI; @@ -33,6 +32,7 @@ import org.archive.modules.CrawlURI; public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { private static final long serialVersionUID = 3L; + @SuppressWarnings("unused") private static final Logger logger = Logger.getLogger(TooManyPathSegmentsDecideRule.class.getName()); /** default for this class is to REJECT */ @@ -69,9 +69,8 @@ public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { protected boolean evaluate(CrawlURI curi) { String uriPath = curi.getUURI().getEscapedPath(); if (uriPath == null) { - uriPath = curi.toString(); - // TODO this is a quick fix. figure out why getUURI().getEscapedPath() returns null. - logger.log(Level.WARNING, "getUURI().getEscapedPath() returns null for the given CrawlURI: "+ uriPath); + // no path means no segments + return false; } int count = 0; int threshold = getMaxPathDepth(); From c5d740a6d43903d4888df492b65852e1c62fee48 Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Wed, 11 Jul 2012 16:57:13 -0700 Subject: [PATCH 6/6] remove some dead or redundant code * ArchiveUtils sb.length() was tested twice and int i was never used * PrefixFinder opCount is never used * BdbMultipleWorkQueues tries is never used --- .../src/main/java/org/archive/util/ArchiveUtils.java | 12 ++++-------- .../src/main/java/org/archive/util/PrefixFinder.java | 4 ---- .../crawler/frontier/BdbMultipleWorkQueues.java | 2 -- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index fce854d1..fe2da96f 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -353,15 +353,11 @@ public class ArchiveUtils { d.length()); } StringBuilder sb = new StringBuilder(d); - if (sb.length() < 8) { - for (int i = sb.length(); sb.length() < 8; i += 2) { - sb.append("01"); - } + while (sb.length() < 8) { + sb.append("01"); } - if (sb.length() < 12) { - for (int i = sb.length(); sb.length() < 12; i++) { - sb.append("0"); - } + while (sb.length() < 12) { + sb.append("0"); } date = ArchiveUtils.parse12DigitDate(sb.toString()); } diff --git a/commons/src/main/java/org/archive/util/PrefixFinder.java b/commons/src/main/java/org/archive/util/PrefixFinder.java index 9d22b4c1..4217e41f 100644 --- a/commons/src/main/java/org/archive/util/PrefixFinder.java +++ b/commons/src/main/java/org/archive/util/PrefixFinder.java @@ -56,9 +56,7 @@ public class PrefixFinder { public static List find(SortedSet set, String input) { LinkedList result = new LinkedList(); set = headSetInclusive(set, input); - int opCount = 0; for (String last = last(set); last != null; last = last(set)) { - opCount++; if (input.startsWith(last)) { result.push(last); set = set.headSet(last); @@ -104,9 +102,7 @@ public class PrefixFinder { public static List findKeys(SortedMap map, String input) { LinkedList result = new LinkedList(); map = headMapInclusive(map, input); - int opCount = 0; for (String last = last(map); last != null; last = last(map)) { - opCount++; if (input.startsWith(last)) { result.push(last); map = map.headMap(last); diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index b94e58d6..e2d80c7d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -153,7 +153,6 @@ public class BdbMultipleWorkQueues { boolean verbose) throws DatabaseException { int matches = 0; - int tries = 0; ArrayList results = new ArrayList(maxMatches); DatabaseEntry key; @@ -184,7 +183,6 @@ public class BdbMultipleWorkQueues { } matches++; } - tries++; } result = cursor.getNext(key,value,null); }