From 3dce1e909d4bfefa6f3c075cf65d9ec33ff75b97 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 5 Jun 2012 17:55:41 -0700 Subject: [PATCH] Change scope of most/all default-scope i.e. package-scoped class members to protected, to make heritrix more friendly to third-party extensions. (I don't think there was ever an intent to limit the scope of any of these to package-only.) --- .../commons/httpclient/HttpMethodBase.java | 2 +- .../main/java/org/archive/bdb/AutoKryo.java | 2 +- .../main/java/org/archive/bdb/BdbModule.java | 22 ++++++------ .../bdb/DisposableStoredSortedMap.java | 2 +- .../java/org/archive/bdb/StoredQueue.java | 8 ++--- .../org/archive/checkpointing/Checkpoint.java | 6 ++-- .../SingleHttpConnectionManager.java | 2 +- .../main/java/org/archive/io/Arc2Warc.java | 4 +-- .../java/org/archive/io/ArchiveRecord.java | 12 +++---- .../java/org/archive/io/CharSubSequence.java | 6 ++-- .../archive/io/GZIPMembersInputStream.java | 10 +++--- .../org/archive/io/HeaderedArchiveRecord.java | 2 +- .../io/ObjectPlusFilesInputStream.java | 4 +-- .../io/ObjectPlusFilesOutputStream.java | 2 +- .../archive/io/RandomAccessOutputStream.java | 2 +- .../java/org/archive/io/ReadSourceEditor.java | 2 +- .../org/archive/io/ReplayInputStream.java | 5 ++- .../main/java/org/archive/io/Warc2Arc.java | 2 +- .../main/java/org/archive/io/WriterPool.java | 4 +-- .../java/org/archive/io/arc/ARCReader.java | 4 +-- .../org/archive/io/arc/ARCRecordMetaData.java | 2 +- .../io/arc/WriterPoolSettingsData.java | 16 ++++----- .../java/org/archive/io/warc/WARCReader.java | 2 +- .../java/org/archive/net/PublicSuffixes.java | 4 +-- .../org/archive/net/s3/S3URLConnection.java | 6 ++-- .../spring/BeanFieldsPatternValidator.java | 10 +++--- .../java/org/archive/spring/ConfigPath.java | 8 ++--- .../archive/spring/ConfigPathConfigurer.java | 8 ++--- .../org/archive/spring/ConfigPathEditor.java | 2 +- .../java/org/archive/spring/ConfigString.java | 2 +- .../spring/HeritrixLifecycleProcessor.java | 2 +- .../org/archive/spring/KeyedProperties.java | 4 +-- .../archive/spring/PathSharingContext.java | 2 +- .../main/java/org/archive/spring/Sheet.java | 6 ++-- .../java/org/archive/util/ArchiveUtils.java | 8 ++--- .../org/archive/util/BloomFilter64bit.java | 8 ++--- .../main/java/org/archive/util/FileUtils.java | 2 +- .../util/IdentityCacheableWrapper.java | 4 +-- .../util/InterruptibleCharSequence.java | 2 +- .../java/org/archive/util/Iteratorable.java | 2 +- .../archive/util/LongToIntConsistentHash.java | 6 ++-- .../util/LongToIntConsistentHashTest.java | 2 +- .../java/org/archive/util/MimetypeUtils.java | 2 +- .../archive/util/ObjectIdentityBdbCache.java | 6 ++-- .../util/ObjectIdentityBdbManualCache.java | 2 +- .../archive/util/ObjectIdentityMemCache.java | 2 +- .../org/archive/util/PaddingStringBuffer.java | 4 +-- .../java/org/archive/util/ProcessUtils.java | 2 +- .../java/org/archive/util/PropertyUtils.java | 2 +- .../main/java/org/archive/util/Recorder.java | 4 +-- .../src/main/java/org/archive/util/SURT.java | 10 +++--- .../main/java/org/archive/util/Supplier.java | 2 +- .../main/java/org/archive/util/UriUtils.java | 6 ++-- .../org/archive/util/anvl/ANVLRecord.java | 4 +-- .../util/bdbje/EnhancedEnvironment.java | 4 +-- .../util/fingerprint/ArrayLongFPCache.java | 6 ++-- .../util/fingerprint/LongFPSetTestCase.java | 2 +- .../util/iterator/CompositeIterator.java | 6 ++-- .../util/ms/DefaultBlockFileSystem.java | 2 +- .../java/org/archive/util/ms/PieceTable.java | 6 ++-- .../util/zip/OpenJDK7InflaterInputStream.java | 2 +- .../util/fingerprint/LongFPSetCacheTest.java | 2 +- .../util/fingerprint/MemLongFPSetTest.java | 2 +- .../crawler/event/StatSnapshotEvent.java | 2 +- .../crawler/framework/ActionDirectory.java | 6 ++-- .../crawler/framework/CheckpointService.java | 14 ++++---- .../framework/CheckpointSuccessEvent.java | 2 +- .../crawler/framework/CrawlController.java | 24 ++++++------- .../archive/crawler/framework/CrawlJob.java | 16 ++++----- .../crawler/framework/CrawlLimitEnforcer.java | 6 ++-- .../org/archive/crawler/framework/Scoper.java | 2 +- .../crawler/frontier/AbstractFrontier.java | 10 +++--- .../archive/crawler/frontier/BdbFrontier.java | 14 ++++---- .../frontier/BdbMultipleWorkQueues.java | 8 ++--- .../frontier/QueueAssignmentPolicy.java | 2 +- .../frontier/RecyclingSerialBinding.java | 2 +- ...RIAuthorityBasedQueueAssignmentPolicy.java | 2 +- .../archive/crawler/frontier/WorkQueue.java | 6 ++-- .../crawler/frontier/WorkQueueFrontier.java | 10 +++--- .../precedence/SimplePrecedenceProvider.java | 2 +- .../crawler/migrate/MigrateH1to3Tool.java | 2 +- .../crawler/monitor/DiskSpaceMonitor.java | 10 +++--- .../postprocessor/DispositionProcessor.java | 2 +- .../postprocessor/LowDiskPauseProcessor.java | 6 ++-- .../prefetch/PreconditionEnforcer.java | 2 +- .../prefetch/RuntimeLimitEnforcer.java | 4 +-- .../crawler/processor/CrawlMapper.java | 16 ++++----- .../crawler/processor/HashCrawlMapper.java | 2 +- .../crawler/processor/LexicalCrawlMapper.java | 6 ++-- .../reporting/CrawlerLoggerModule.java | 6 ++-- .../crawler/reporting/StatisticsTracker.java | 34 +++++++++---------- .../crawler/restlet/BeanBrowseResource.java | 4 +-- .../crawler/restlet/EditRepresentation.java | 4 +-- .../crawler/restlet/EngineApplication.java | 6 ++-- .../archive/crawler/restlet/EnhDirectory.java | 8 ++--- .../org/archive/crawler/restlet/Flash.java | 4 +-- .../crawler/restlet/JobRelatedResource.java | 6 ++-- .../crawler/restlet/PagedRepresentation.java | 16 ++++----- .../crawler/restlet/RateLimitGuard.java | 2 +- .../crawler/restlet/ReportGenResource.java | 2 +- .../crawler/restlet/ScriptResource.java | 16 ++++----- .../spring/DecideRuledSheetAssociation.java | 6 ++-- .../crawler/spring/SheetAssociation.java | 2 +- .../crawler/spring/SheetOverlaysManager.java | 8 ++--- .../spring/SurtPrefixesSheetAssociation.java | 2 +- .../crawler/util/BdbUriUniqFilter.java | 6 ++-- .../crawler/util/BenchmarkUriUniqFilters.java | 2 +- .../crawler/util/BloomUriUniqFilter.java | 2 +- .../util/DiskFPMergeUriUniqFilter.java | 14 ++++---- .../archive/crawler/util/FPUriUniqFilter.java | 2 +- .../java/org/archive/crawler/util/Logs.java | 2 +- .../crawler/util/MemUriUniqFilter.java | 2 +- .../org/archive/crawler/util/TopNSet.java | 12 +++---- .../org/archive/modules/CrawlMetadata.java | 16 ++++----- .../java/org/archive/modules/CrawlURI.java | 29 ++++------------ .../java/org/archive/modules/Processor.java | 4 +-- .../org/archive/modules/ProcessorChain.java | 4 +-- .../archive/modules/ScriptedProcessor.java | 4 +-- .../modules/credential/Credential.java | 2 +- .../modules/credential/CredentialStore.java | 2 +- .../credential/HtmlFormCredential.java | 6 ++-- .../HttpAuthenticationCredential.java | 6 ++-- .../ExternalGeoLocationDecideRule.java | 4 +-- .../deciderules/FetchStatusDecideRule.java | 2 +- .../ResourceNoLongerThanDecideRule.java | 2 +- .../deciderules/ScriptedDecideRule.java | 2 +- .../extractor/AggressiveExtractorHTML.java | 2 +- .../modules/extractor/CustomSWFTags.java | 2 +- .../archive/modules/extractor/Extractor.java | 2 +- .../modules/extractor/ExtractorCSS.java | 4 +-- .../modules/extractor/ExtractorHTML.java | 2 +- .../modules/extractor/ExtractorHTTP.java | 2 +- .../modules/extractor/ExtractorJS.java | 2 +- .../modules/extractor/ExtractorSWF.java | 4 +-- .../modules/extractor/ExtractorURI.java | 2 +- .../modules/extractor/ExtractorUniversal.java | 2 +- .../extractor/JerichoExtractorHTML.java | 2 +- .../archive/modules/extractor/PDFParser.java | 12 +++---- .../fetcher/AbstractCookieStorage.java | 2 +- .../modules/fetcher/BdbCookieStorage.java | 2 +- .../org/archive/modules/fetcher/FetchDNS.java | 4 +-- .../org/archive/modules/fetcher/FetchFTP.java | 2 +- .../archive/modules/fetcher/FetchHTTP.java | 4 +-- .../archive/modules/fetcher/FetchStats.java | 32 ++++++++--------- .../archive/modules/net/BdbServerCache.java | 2 +- .../org/archive/modules/net/CrawlServer.java | 6 ++-- .../modules/net/CustomRobotsPolicy.java | 6 ++-- .../modules/net/DefaultTempDirProvider.java | 5 +-- .../modules/net/FirstNamedRobotsPolicy.java | 6 ++-- .../modules/net/MostFavoredRobotsPolicy.java | 6 ++-- .../archive/modules/net/RobotsDirectives.java | 6 ++-- .../org/archive/modules/net/Robotstxt.java | 10 +++--- .../recrawl/FetchHistoryProcessor.java | 2 +- .../modules/recrawl/PersistLoadProcessor.java | 4 +-- .../modules/recrawl/PersistLogProcessor.java | 2 +- .../recrawl/PersistOnlineProcessor.java | 2 +- .../org/archive/modules/seeds/SeedModule.java | 2 +- .../modules/writer/Kw3WriterProcessor.java | 12 +++---- .../modules/writer/MirrorWriterProcessor.java | 30 ++++++++-------- .../modules/writer/WARCWriterProcessor.java | 2 +- .../modules/writer/WriterPoolProcessor.java | 22 ++++++------ 161 files changed, 454 insertions(+), 473 deletions(-) diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java index 41644817..fe8be4b0 100644 --- a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java +++ b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java @@ -2360,7 +2360,7 @@ public abstract class HttpMethodBase implements HttpMethod { * * TODO: Remove this crap as soon as possible */ - void fakeResponse( + protected void fakeResponse( StatusLine statusline, HeaderGroup responseheaders, InputStream responseStream diff --git a/commons/src/main/java/org/archive/bdb/AutoKryo.java b/commons/src/main/java/org/archive/bdb/AutoKryo.java index 2c6a89fa..1306beec 100644 --- a/commons/src/main/java/org/archive/bdb/AutoKryo.java +++ b/commons/src/main/java/org/archive/bdb/AutoKryo.java @@ -26,7 +26,7 @@ import com.esotericsoftware.kryo.SerializationException; */ @SuppressWarnings("unchecked") public class AutoKryo extends Kryo { - ArrayList registeredClasses = new ArrayList(); + protected ArrayList registeredClasses = new ArrayList(); @Override protected void handleUnregisteredClass(Class type) { diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 19c20f68..10b0d2ba 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -94,10 +94,10 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab public static class BdbConfig implements Serializable { private static final long serialVersionUID = 1L; - boolean allowCreate; - boolean sortedDuplicates; - boolean transactional; - boolean deferredWrite = true; + protected boolean allowCreate; + protected boolean sortedDuplicates; + protected boolean transactional; + protected boolean deferredWrite = true; public BdbConfig() { } @@ -148,7 +148,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab this.dir = dir; } - int cachePercent = -1; + protected int cachePercent = -1; public int getCachePercent() { return cachePercent; } @@ -156,7 +156,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab this.cachePercent = cachePercent; } - int cacheSize = -1; + protected int cacheSize = -1; public int getCacheSize() { return cacheSize; } @@ -164,7 +164,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab this.cacheSize = cacheSize; } - boolean useSharedCache = true; + protected boolean useSharedCache = true; public boolean getUseSharedCache() { return useSharedCache; } @@ -177,7 +177,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab * according to JE FAQ * http://www.oracle.com/technology/products/berkeley-db/faq/je_faq.html#33 */ - int expectedConcurrency = 64; + protected int expectedConcurrency = 64; public int getExpectedConcurrency() { return expectedConcurrency; } @@ -196,7 +196,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab * in any checkpoint's 'jdbfiles.manifest' should be retained to * keep the checkpoint valid. */ - boolean useHardLinkCheckpoints = true; + protected boolean useHardLinkCheckpoints = true; public boolean getUseHardLinkCheckpoints() { return useHardLinkCheckpoints; } @@ -587,7 +587,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab public void finishCheckpoint(Checkpoint checkpointInProgress) {} - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; @Autowired(required=false) public void setRecoveryCheckpoint(Checkpoint checkpoint) { this.recoveryCheckpoint = checkpoint; @@ -630,7 +630,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab } /** uniqueness serial number for temp map databases */ - long sn = 0; + protected long sn = 0; /** * Creates a database-backed TempStoredSortedMap for transient diff --git a/commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java b/commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java index efae297c..9c276e48 100644 --- a/commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java +++ b/commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java @@ -43,7 +43,7 @@ import com.sleepycat.je.DatabaseException; public class DisposableStoredSortedMap extends StoredSortedMap { final private static Logger LOGGER = Logger.getLogger(DisposableStoredSortedMap.class.getName()); - Database db; + protected Database db; public DisposableStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, boolean arg3) { super(db, arg1, arg2, arg3); diff --git a/commons/src/main/java/org/archive/bdb/StoredQueue.java b/commons/src/main/java/org/archive/bdb/StoredQueue.java index 0009db8b..767d77a7 100644 --- a/commons/src/main/java/org/archive/bdb/StoredQueue.java +++ b/commons/src/main/java/org/archive/bdb/StoredQueue.java @@ -46,10 +46,10 @@ public class StoredQueue extends AbstractQueue { private static final Logger logger = Logger.getLogger(StoredQueue.class.getName()); - transient StoredSortedMap queueMap; // Long -> E - transient Database queueDb; // Database - AtomicLong tailIndex; // next spot for insert - transient volatile E peekItem = null; + protected transient StoredSortedMap queueMap; // Long -> E + protected transient Database queueDb; // Database + protected AtomicLong tailIndex; // next spot for insert + protected transient volatile E peekItem = null; /** * Create a StoredQueue backed by the given Database. diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java index 47cda1ca..3daf8a73 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -50,9 +50,9 @@ public class Checkpoint implements InitializingBean { /** Name of file written with timestamp into valid checkpoints */ public static final String VALIDITY_STAMP_FILENAME = "valid"; - String name; - String shortName; - boolean success = false; + protected String name; + protected String shortName; + protected boolean success = false; /** * Checkpoints directory; either an absolute path, or relative to the diff --git a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java index be8787a6..4ba6a837 100644 --- a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java +++ b/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java @@ -54,7 +54,7 @@ public class SingleHttpConnectionManager extends SimpleHttpConnectionManager { finishLast(conn); } - static void finishLast(HttpConnection conn) { + protected static void finishLast(HttpConnection conn) { // copied from superclass because it wasn't made available to subclasses InputStream lastResponse = conn.getLastResponseInputStream(); if (lastResponse != null) { diff --git a/commons/src/main/java/org/archive/io/Arc2Warc.java b/commons/src/main/java/org/archive/io/Arc2Warc.java index 7c18a80d..5cd99c95 100644 --- a/commons/src/main/java/org/archive/io/Arc2Warc.java +++ b/commons/src/main/java/org/archive/io/Arc2Warc.java @@ -58,9 +58,9 @@ import org.joda.time.format.ISODateTimeFormat; * @version $Date$ $Revision$ */ public class Arc2Warc { - RecordIDGenerator generator = new UUIDGenerator(); + protected RecordIDGenerator generator = new UUIDGenerator(); - private static void usage(HelpFormatter formatter, Options options, + private static void usage(HelpFormatter formatter, Options options, int exitCode) { formatter.printHelp("java org.archive.io.arc.Arc2Warc " + "[--force] ARC_INPUT WARC_OUTPUT", options); diff --git a/commons/src/main/java/org/archive/io/ArchiveRecord.java b/commons/src/main/java/org/archive/io/ArchiveRecord.java index 3243ca7e..48bb8ae0 100644 --- a/commons/src/main/java/org/archive/io/ArchiveRecord.java +++ b/commons/src/main/java/org/archive/io/ArchiveRecord.java @@ -42,7 +42,7 @@ public abstract class ArchiveRecord extends InputStream { protected static final long MIN_HTTP_HEADER_LENGTH = Math.min("HTTP/1.1 200 OK\r\n".length(), "GET / HTTP/1.0\n\r".length()); - ArchiveRecordHeader header = null; + protected ArchiveRecordHeader header = null; /** * Stream to read this record from. @@ -56,19 +56,19 @@ public abstract class ArchiveRecord extends InputStream { * the underlying stream is managing our not reading too much (This pertains * to the skipping over the end of the ARCRecord. See {@link #skip()}. */ - InputStream in = null; + protected InputStream in = null; /** * Position w/i the Record content, within in. * This position is relative within this Record. Its not same as the * Archive file position. */ - long position = 0; + protected long position = 0; /** * Set flag when we've reached the end-of-record. */ - boolean eor = false; + protected boolean eor = false; /** * Compute digest on what we read and add to metadata when done. @@ -84,7 +84,7 @@ public abstract class ArchiveRecord extends InputStream { protected MessageDigest digest = null; private String digestStr = null; - boolean strict = false; + protected boolean strict = false; /** @@ -235,7 +235,7 @@ public abstract class ArchiveRecord extends InputStream { * * @throws IOException */ - void skip() throws IOException { + protected void skip() throws IOException { if (this.eor) { return; } diff --git a/commons/src/main/java/org/archive/io/CharSubSequence.java b/commons/src/main/java/org/archive/io/CharSubSequence.java index 1e32c2b4..1e89da56 100644 --- a/commons/src/main/java/org/archive/io/CharSubSequence.java +++ b/commons/src/main/java/org/archive/io/CharSubSequence.java @@ -27,9 +27,9 @@ package org.archive.io; */ public class CharSubSequence implements CharSequence { - CharSequence inner; - int start; - int end; + protected CharSequence inner; + protected int start; + protected int end; public CharSubSequence(CharSequence inner, int start, int end) { if (end < start) { diff --git a/commons/src/main/java/org/archive/io/GZIPMembersInputStream.java b/commons/src/main/java/org/archive/io/GZIPMembersInputStream.java index 7f108d3b..ca2ce07d 100644 --- a/commons/src/main/java/org/archive/io/GZIPMembersInputStream.java +++ b/commons/src/main/java/org/archive/io/GZIPMembersInputStream.java @@ -66,11 +66,11 @@ import com.google.common.io.CountingInputStream; * @contributor gojomo */ public class GZIPMembersInputStream extends OpenJDK7GZIPInputStream { - long memberNumber = 0; - long holdAtMemberNumber = Long.MAX_VALUE; - long currentMemberStart = 0; - long currentMemberEnd = -1; - InputStream originalIn; + protected long memberNumber = 0; + protected long holdAtMemberNumber = Long.MAX_VALUE; + protected long currentMemberStart = 0; + protected long currentMemberEnd = -1; + protected InputStream originalIn; public GZIPMembersInputStream(InputStream in) throws IOException { this(in,512); diff --git a/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java b/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java index 2a669a84..ce339b6b 100644 --- a/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java +++ b/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java @@ -411,7 +411,7 @@ public class HeaderedArchiveRecord extends ArchiveRecord { } @Override - void skip() throws IOException { + protected void skip() throws IOException { ((ArchiveRecord)this.in).skip(); } diff --git a/commons/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java b/commons/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java index 341c4789..892860ed 100644 --- a/commons/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java +++ b/commons/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java @@ -38,8 +38,8 @@ import org.archive.util.FileUtils; * */ public class ObjectPlusFilesInputStream extends ObjectInputStream { - LinkedList auxiliaryDirectoryStack = new LinkedList(); - LinkedList postRestoreTasks = new LinkedList(); + protected LinkedList auxiliaryDirectoryStack = new LinkedList(); + protected LinkedList postRestoreTasks = new LinkedList(); /** * Instantiate over the given stream and using the supplied diff --git a/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java b/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java index 3449b891..224f24e7 100644 --- a/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java +++ b/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java @@ -35,7 +35,7 @@ import org.archive.util.FileUtils; * @author gojomo */ public class ObjectPlusFilesOutputStream extends ObjectOutputStream { - LinkedList auxiliaryDirectoryStack = new LinkedList(); + protected LinkedList auxiliaryDirectoryStack = new LinkedList(); /** * Constructor diff --git a/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java b/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java index e0f52fe6..225f995f 100644 --- a/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java +++ b/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java @@ -29,7 +29,7 @@ import java.io.RandomAccessFile; * @author gojomo */ public class RandomAccessOutputStream extends OutputStream { - RandomAccessFile raf; + protected RandomAccessFile raf; /** * Wrap the given RandomAccessFile diff --git a/commons/src/main/java/org/archive/io/ReadSourceEditor.java b/commons/src/main/java/org/archive/io/ReadSourceEditor.java index 4185d8f8..23f76971 100644 --- a/commons/src/main/java/org/archive/io/ReadSourceEditor.java +++ b/commons/src/main/java/org/archive/io/ReadSourceEditor.java @@ -34,7 +34,7 @@ import org.archive.spring.ConfigString; * future ReadSource options) might otherwise occur. */ public class ReadSourceEditor implements PropertyEditor { - Object value; + protected Object value; public void addPropertyChangeListener(PropertyChangeListener listener) { // TODO Auto-generated method stub diff --git a/commons/src/main/java/org/archive/io/ReplayInputStream.java b/commons/src/main/java/org/archive/io/ReplayInputStream.java index 8a743558..fccf5fd3 100644 --- a/commons/src/main/java/org/archive/io/ReplayInputStream.java +++ b/commons/src/main/java/org/archive/io/ReplayInputStream.java @@ -104,7 +104,7 @@ public class ReplayInputStream extends SeekInputStream diskStream = new BufferedSeekInputStream(rais, 4096); } - File backingFile; + protected File backingFile; /** * Create a ReplayInputStream from the given source stream. Requires @@ -319,8 +319,7 @@ public class ReplayInputStream extends SeekInputStream return position; } - // package private - byte[] getBuffer() { + protected byte[] getBuffer() { return buffer; } } diff --git a/commons/src/main/java/org/archive/io/Warc2Arc.java b/commons/src/main/java/org/archive/io/Warc2Arc.java index 6156fefb..eb611cea 100644 --- a/commons/src/main/java/org/archive/io/Warc2Arc.java +++ b/commons/src/main/java/org/archive/io/Warc2Arc.java @@ -64,7 +64,7 @@ public class Warc2Arc { System.exit(exitCode); } - static String parseRevision(final String version) { + protected static String parseRevision(final String version) { final String ID = "$Revision: "; int index = version.indexOf(ID); return (index < 0)? version: diff --git a/commons/src/main/java/org/archive/io/WriterPool.java b/commons/src/main/java/org/archive/io/WriterPool.java index 7bf35cfa..79dad027 100644 --- a/commons/src/main/java/org/archive/io/WriterPool.java +++ b/commons/src/main/java/org/archive/io/WriterPool.java @@ -37,7 +37,7 @@ import java.util.logging.Logger; * @author stack */ public abstract class WriterPool { - final Logger logger = Logger.getLogger(this.getClass().getName()); + private final Logger logger = Logger.getLogger(this.getClass().getName()); /** * Used to generate unique filename sequences. @@ -72,7 +72,7 @@ public abstract class WriterPool { /** current count of active writers; only read/mutated in synchronized blocks */ protected int currentActive = 0; /** round-robin queue of available writers */ - BlockingQueue availableWriters; + protected BlockingQueue availableWriters; /** system time when writer was last wanted (because one was not ready in time) */ protected long lastWriterNeededTime; diff --git a/commons/src/main/java/org/archive/io/arc/ARCReader.java b/commons/src/main/java/org/archive/io/arc/ARCReader.java index 4ec5497b..fcb6c15b 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCReader.java +++ b/commons/src/main/java/org/archive/io/arc/ARCReader.java @@ -67,7 +67,7 @@ import org.archive.util.ArchiveUtils; */ public abstract class ARCReader extends ArchiveReader implements ARCConstants, Closeable { - Logger logger = Logger.getLogger(ARCReader.class.getName()); + private final Logger logger = Logger.getLogger(ARCReader.class.getName()); /** * Set to true if we are aligned on first record of Archive file. @@ -80,7 +80,7 @@ implements ARCConstants, Closeable { private boolean parseHttpHeaders = true; - ARCReader() { + protected ARCReader() { super(); } diff --git a/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java b/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java index 95a3003c..2b541196 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java +++ b/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java @@ -253,7 +253,7 @@ public class ARCRecordMetaData implements ArchiveRecordHeader, ARCConstants { return this.contentBegin; } - void setContentBegin(final int offset) { + protected void setContentBegin(final int offset) { this.contentBegin = offset; } } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java b/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java index 33490b4a..7396f2d8 100644 --- a/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java +++ b/commons/src/main/java/org/archive/io/arc/WriterPoolSettingsData.java @@ -24,14 +24,14 @@ import java.util.List; import org.archive.io.WriterPoolSettings; public class WriterPoolSettingsData implements WriterPoolSettings { - long maxFileSizeBytes; - String prefix; - String template; - List outputDirs; - boolean compress; - List metadata; - boolean frequentFlushes = true; - int writeBufferSize = 16*1024; + protected long maxFileSizeBytes; + protected String prefix; + protected String template; + protected List outputDirs; + protected boolean compress; + protected List metadata; + protected boolean frequentFlushes = true; + protected int writeBufferSize = 16*1024; public WriterPoolSettingsData(String prefix, String template, long maxFileSizeBytes, boolean compress, List outputDirs, diff --git a/commons/src/main/java/org/archive/io/warc/WARCReader.java b/commons/src/main/java/org/archive/io/warc/WARCReader.java index 74de4929..3794a23d 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCReader.java +++ b/commons/src/main/java/org/archive/io/warc/WARCReader.java @@ -43,7 +43,7 @@ import org.archive.io.warc.WARCConstants; * @version $Date: 2006-11-27 18:03:03 -0800 (Mon, 27 Nov 2006) $ $Version$ */ public class WARCReader extends ArchiveReader implements WARCConstants { - WARCReader() { + protected WARCReader() { super(); } diff --git a/commons/src/main/java/org/archive/net/PublicSuffixes.java b/commons/src/main/java/org/archive/net/PublicSuffixes.java index 4165d69c..eab8081a 100644 --- a/commons/src/main/java/org/archive/net/PublicSuffixes.java +++ b/commons/src/main/java/org/archive/net/PublicSuffixes.java @@ -85,8 +85,8 @@ public class PublicSuffixes { * {@code cs} must not be null except for root node. */ public static class Node implements Comparable { - CharSequence cs; - List branches; + protected CharSequence cs; + protected List branches; public Node() { this("", null); } diff --git a/commons/src/main/java/org/archive/net/s3/S3URLConnection.java b/commons/src/main/java/org/archive/net/s3/S3URLConnection.java index 668dfb43..63c63a99 100644 --- a/commons/src/main/java/org/archive/net/s3/S3URLConnection.java +++ b/commons/src/main/java/org/archive/net/s3/S3URLConnection.java @@ -17,10 +17,10 @@ import org.jets3t.service.S3ServiceException; * @author jlee */ public class S3URLConnection extends URLConnection { - String id = ""; - String secret = ""; + protected String id = ""; + protected String secret = ""; - S3Object object = null; + protected S3Object object = null; /** * Contruct a new S3URLConnection. diff --git a/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java b/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java index 84a7aafb..7e530141 100644 --- a/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java +++ b/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java @@ -30,9 +30,9 @@ import org.springframework.validation.Validator; public class BeanFieldsPatternValidator implements Validator { public class PropertyPatternRule { - String propertyName; - Pattern requiredPattern; - String errorMessage; + protected String propertyName; + protected Pattern requiredPattern; + protected String errorMessage; public PropertyPatternRule(String name, String pat, String msg) { propertyName = name; requiredPattern = Pattern.compile(pat); @@ -48,8 +48,8 @@ public class BeanFieldsPatternValidator implements Validator { } - Class clazz; - List rules; + protected Class clazz; + protected List rules; public BeanFieldsPatternValidator(Class clazz, String ... fieldsPatterns) { this.clazz = clazz; diff --git a/commons/src/main/java/org/archive/spring/ConfigPath.java b/commons/src/main/java/org/archive/spring/ConfigPath.java index 142994f7..276a7ac4 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPath.java +++ b/commons/src/main/java/org/archive/spring/ConfigPath.java @@ -54,9 +54,9 @@ import org.springframework.beans.factory.annotation.Required; public class ConfigPath implements Serializable { private static final long serialVersionUID = 1L; - String name; - String path; - ConfigPath base; + protected String name; + protected String path; + protected ConfigPath base; public ConfigPath() { super(); @@ -124,7 +124,7 @@ public class ConfigPath implements Serializable { return this; } - ConfigPathConfigurer configurer; + protected ConfigPathConfigurer configurer; public void setConfigurer(ConfigPathConfigurer configPathConfigurer) { this.configurer = configPathConfigurer; } diff --git a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java index a572d4fa..4074b7e2 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java +++ b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java @@ -61,14 +61,14 @@ implements private static final Logger logger = Logger.getLogger(ConfigPathConfigurer.class.getName()); - Map allBeans = new HashMap(); + protected Map allBeans = new HashMap(); //// BEAN PROPERTIES /** 'home' directory for all other paths to be resolved * relative to; defaults to directory of primary XML config file */ - ConfigPath path; + protected ConfigPath path; public ConfigPath getPath() { return path; } @@ -150,7 +150,7 @@ implements //// APPLICATIONCONTEXTAWARE IMPLEMENTATION - PathSharingContext appCtx; + protected PathSharingContext appCtx; /** * Remember ApplicationContext, and if possible primary * configuration file's home directory. @@ -172,7 +172,7 @@ implements } // REMEMBERED CONFIGPATHS - Map allConfigPaths = new HashMap(); + protected Map allConfigPaths = new HashMap(); protected void remember(String key, ConfigPath cp) { allConfigPaths.put(key, cp); } diff --git a/commons/src/main/java/org/archive/spring/ConfigPathEditor.java b/commons/src/main/java/org/archive/spring/ConfigPathEditor.java index 79b8bd4f..03b63ca1 100644 --- a/commons/src/main/java/org/archive/spring/ConfigPathEditor.java +++ b/commons/src/main/java/org/archive/spring/ConfigPathEditor.java @@ -30,7 +30,7 @@ import java.beans.PropertyEditor; * */ public class ConfigPathEditor implements PropertyEditor { - Object value; + protected Object value; public void addPropertyChangeListener(PropertyChangeListener listener) { // TODO Auto-generated method stub diff --git a/commons/src/main/java/org/archive/spring/ConfigString.java b/commons/src/main/java/org/archive/spring/ConfigString.java index 30cd0488..46f1d571 100644 --- a/commons/src/main/java/org/archive/spring/ConfigString.java +++ b/commons/src/main/java/org/archive/spring/ConfigString.java @@ -32,7 +32,7 @@ import org.archive.io.ReadSource; * @contributor gojomo */ public class ConfigString implements ReadSource { - String value; + protected String value; public String getValue() { return value; } diff --git a/commons/src/main/java/org/archive/spring/HeritrixLifecycleProcessor.java b/commons/src/main/java/org/archive/spring/HeritrixLifecycleProcessor.java index 7e349c52..af04745e 100644 --- a/commons/src/main/java/org/archive/spring/HeritrixLifecycleProcessor.java +++ b/commons/src/main/java/org/archive/spring/HeritrixLifecycleProcessor.java @@ -30,7 +30,7 @@ import org.springframework.context.support.DefaultLifecycleProcessor; * @contributor gojomo */ public class HeritrixLifecycleProcessor extends DefaultLifecycleProcessor implements BeanNameAware { - String name; + protected String name; @Override public void onRefresh() { diff --git a/commons/src/main/java/org/archive/spring/KeyedProperties.java b/commons/src/main/java/org/archive/spring/KeyedProperties.java index 0a5b562b..4651e83a 100644 --- a/commons/src/main/java/org/archive/spring/KeyedProperties.java +++ b/commons/src/main/java/org/archive/spring/KeyedProperties.java @@ -42,7 +42,7 @@ public class KeyedProperties extends ConcurrentHashMap { /** the alternate global property-paths leading to this map * TODO: consider if deterministic ordered list is important */ - HashSet externalPaths = new HashSet(); + protected HashSet externalPaths = new HashSet(); /** * Add a path by which the outside world can reach this map @@ -104,7 +104,7 @@ public class KeyedProperties extends ConcurrentHashMap { /** * ThreadLocal (contextual) collection of pushed override maps */ - static ThreadLocal> threadOverrides = + protected static ThreadLocal> threadOverrides = new ThreadLocal>() { protected ArrayList initialValue() { return new ArrayList(); diff --git a/commons/src/main/java/org/archive/spring/PathSharingContext.java b/commons/src/main/java/org/archive/spring/PathSharingContext.java index e863773a..9964f555 100644 --- a/commons/src/main/java/org/archive/spring/PathSharingContext.java +++ b/commons/src/main/java/org/archive/spring/PathSharingContext.java @@ -88,7 +88,7 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { // // Cascading self-validation // - HashMap allErrors; // bean name -> Errors + protected HashMap allErrors; // bean name -> Errors public void validate() { allErrors = new HashMap(); diff --git a/commons/src/main/java/org/archive/spring/Sheet.java b/commons/src/main/java/org/archive/spring/Sheet.java index c9458bd4..1153fec7 100644 --- a/commons/src/main/java/org/archive/spring/Sheet.java +++ b/commons/src/main/java/org/archive/spring/Sheet.java @@ -56,13 +56,13 @@ public class Sheet implements BeanFactoryAware, BeanNameAware { * also be another string, in the case of Sheets added after * initial container wiring */ - String name; - BeanFactory beanFactory; + protected String name; + protected BeanFactory beanFactory; /** map of full property-paths (from BeanFactory to individual * property) and their changed value when this Sheet of overrides * is in effect */ - Map map = new ConcurrentHashMap(); + protected Map map = new ConcurrentHashMap(); public void setBeanFactory(BeanFactory beanFactory) throws BeansException { this.beanFactory = beanFactory; diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index db3db1f0..12bc67ac 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -128,8 +128,8 @@ public class ArchiveUtils { return TIMESTAMP17.get().format(new Date()); } - static long LAST_UNIQUE_NOW17 = 0; - static String LAST_TIMESTAMP17 = ""; + protected static long LAST_UNIQUE_NOW17 = 0; + protected static String LAST_TIMESTAMP17 = ""; /** * Utility function for creating UNIQUE-from-this-class * arc-style date stamps in the format yyyMMddHHmmssSSS. @@ -163,8 +163,8 @@ public class ArchiveUtils { return TIMESTAMP14.get().format(new Date()); } - static long LAST_UNIQUE_NOW14 = 0; - static String LAST_TIMESTAMP14 = ""; + protected static long LAST_UNIQUE_NOW14 = 0; + protected static String LAST_TIMESTAMP14 = ""; /** * Utility function for creating UNIQUE-from-this-class * arc-style date stamps in the format yyyMMddHHmmss. diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java index aeca26ed..71899b22 100644 --- a/commons/src/main/java/org/archive/util/BloomFilter64bit.java +++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java @@ -81,7 +81,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter { private static final long serialVersionUID = 2L; /** The number of weights used to create hash functions. */ - final static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + protected final static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 /** The number of bits in this filter. */ final protected long m; /** if bitfield is an exact power of 2 in length, it is this power */ @@ -99,10 +99,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter { * smaller than the actual number of additions of distinct character * sequences because of false positives. */ - int size; + protected int size; /** The natural logarithm of 2, used in the computation of the number of bits. */ - final static double NATURAL_LOG_OF_2 = Math.log( 2 ); + protected final static double NATURAL_LOG_OF_2 = Math.log( 2 ); /** power-of-two to use as maximum size of bitfield subarrays */ protected final static int SUBARRAY_POWER_OF_TWO = 26; // 512MiB of longs @@ -111,7 +111,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter { /** mask for lowest SUBARRAY_POWER_OF_TWO bits */ protected final static int SUBARRAY_MASK = SUBARRAY_LENGTH_IN_LONGS - 1; //0x0FFFFFFF - final static boolean DEBUG = false; + protected final static boolean DEBUG = false; /** Creates a new Bloom filter with given number of hash functions and * expected number of elements. diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java index 595ca475..3b5b93ad 100644 --- a/commons/src/main/java/org/archive/util/FileUtils.java +++ b/commons/src/main/java/org/archive/util/FileUtils.java @@ -600,7 +600,7 @@ public class FileUtils { LOGGER.severe(">50 pending Files to delete even after gc/finalization"); } } - static LinkedList pendingDeletes = new LinkedList(); + protected static LinkedList pendingDeletes = new LinkedList(); /** * Read the entire stream to EOF into the passed file. diff --git a/commons/src/main/java/org/archive/util/IdentityCacheableWrapper.java b/commons/src/main/java/org/archive/util/IdentityCacheableWrapper.java index d7e8a26f..126aad55 100644 --- a/commons/src/main/java/org/archive/util/IdentityCacheableWrapper.java +++ b/commons/src/main/java/org/archive/util/IdentityCacheableWrapper.java @@ -29,7 +29,7 @@ import org.archive.bdb.AutoKryo; public class IdentityCacheableWrapper implements IdentityCacheable { private static final long serialVersionUID = 1L; - K wrapped; + protected K wrapped; public IdentityCacheableWrapper(String key, K wrapped) { super(); @@ -45,7 +45,7 @@ public class IdentityCacheableWrapper implements IdentityCacheable { // IdentityCacheable support // transient private ObjectIdentityCache cache; - String key; + protected String key; @Override public String getKey() { return key; diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java index a6442289..f608c49c 100644 --- a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java +++ b/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java @@ -25,7 +25,7 @@ package org.archive.util; * @author gojomo */ public class InterruptibleCharSequence implements CharSequence { - CharSequence inner; + protected CharSequence inner; // public long counter = 0; public InterruptibleCharSequence(CharSequence inner) { diff --git a/commons/src/main/java/org/archive/util/Iteratorable.java b/commons/src/main/java/org/archive/util/Iteratorable.java index 3bd77689..8d9a6f8d 100644 --- a/commons/src/main/java/org/archive/util/Iteratorable.java +++ b/commons/src/main/java/org/archive/util/Iteratorable.java @@ -27,7 +27,7 @@ import java.util.Iterator; * */ public class Iteratorable implements Iterable { - Iterator iterator; + protected Iterator iterator; public Iteratorable(Iterator iterator) { this.iterator = iterator; diff --git a/commons/src/main/java/org/archive/util/LongToIntConsistentHash.java b/commons/src/main/java/org/archive/util/LongToIntConsistentHash.java index fa55c3c8..ebb2418c 100644 --- a/commons/src/main/java/org/archive/util/LongToIntConsistentHash.java +++ b/commons/src/main/java/org/archive/util/LongToIntConsistentHash.java @@ -32,9 +32,9 @@ import st.ata.util.FPGenerator; */ public class LongToIntConsistentHash { protected static final int DEFAULT_REPLICAS = 128; - TreeMap circle = new TreeMap(); - int replicasInstalledUpTo=-1; - int numReplicas; + protected TreeMap circle = new TreeMap(); + protected int replicasInstalledUpTo=-1; + protected int numReplicas; public LongToIntConsistentHash() { this(DEFAULT_REPLICAS); diff --git a/commons/src/main/java/org/archive/util/LongToIntConsistentHashTest.java b/commons/src/main/java/org/archive/util/LongToIntConsistentHashTest.java index fcc36642..8147a005 100644 --- a/commons/src/main/java/org/archive/util/LongToIntConsistentHashTest.java +++ b/commons/src/main/java/org/archive/util/LongToIntConsistentHashTest.java @@ -26,7 +26,7 @@ import org.apache.commons.lang.math.RandomUtils; import st.ata.util.FPGenerator; public class LongToIntConsistentHashTest extends TestCase { - LongToIntConsistentHash conhash; + protected LongToIntConsistentHash conhash; @Override protected void setUp() throws Exception { diff --git a/commons/src/main/java/org/archive/util/MimetypeUtils.java b/commons/src/main/java/org/archive/util/MimetypeUtils.java index 2b36215d..adfa1a0f 100644 --- a/commons/src/main/java/org/archive/util/MimetypeUtils.java +++ b/commons/src/main/java/org/archive/util/MimetypeUtils.java @@ -37,7 +37,7 @@ public class MimetypeUtils { /** * Truncation regex. */ - final static Pattern TRUNCATION_REGEX = Pattern.compile("^([^\\s;,]+).*"); + protected final static Pattern TRUNCATION_REGEX = Pattern.compile("^([^\\s;,]+).*"); /** diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java index 29526c5e..0a6d1099 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java @@ -323,7 +323,7 @@ implements ObjectIdentityCache, Closeable, Serializable { } } - String composeCacheSummary() { + protected String composeCacheSummary() { long totalHits = cacheHit.get() + diskHit.get(); if (totalHits < 1) { return ""; @@ -427,7 +427,7 @@ implements ObjectIdentityCache, Closeable, Serializable { * Package-protected for unit-test visibility. */ @SuppressWarnings("unchecked") - synchronized void pageOutStaleEntries() { + protected synchronized void pageOutStaleEntries() { int c = 0; long startTime = System.currentTimeMillis(); for(SoftEntry entry; (entry = (SoftEntry)refQueue.poll()) != null;) { @@ -486,7 +486,7 @@ implements ObjectIdentityCache, Closeable, Serializable { } private static class PhantomEntry extends PhantomReference { - final String key; + protected final String key; public PhantomEntry(String key, V referent) { super(referent, null); diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 87298a02..70cd213c 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -264,7 +264,7 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< } } - String composeCacheSummary() { + protected String composeCacheSummary() { long totalHits = cacheHit.get() + diskHit.get(); if (totalHits < 1) { return ""; diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java index deaa7393..f2eceebc 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java @@ -32,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap; */ public class ObjectIdentityMemCache implements ObjectIdentityCache { - ConcurrentHashMap map; + protected ConcurrentHashMap map; public ObjectIdentityMemCache() { map = new ConcurrentHashMap(); diff --git a/commons/src/main/java/org/archive/util/PaddingStringBuffer.java b/commons/src/main/java/org/archive/util/PaddingStringBuffer.java index c7c2d983..fa795807 100644 --- a/commons/src/main/java/org/archive/util/PaddingStringBuffer.java +++ b/commons/src/main/java/org/archive/util/PaddingStringBuffer.java @@ -36,9 +36,9 @@ package org.archive.util; */ public final class PaddingStringBuffer { // The buffer. - StringBuffer buffer; + protected StringBuffer buffer; // Location in current line - int linePos; + protected int linePos; /** * Create a new PaddingStringBuffer diff --git a/commons/src/main/java/org/archive/util/ProcessUtils.java b/commons/src/main/java/org/archive/util/ProcessUtils.java index da827090..af792981 100644 --- a/commons/src/main/java/org/archive/util/ProcessUtils.java +++ b/commons/src/main/java/org/archive/util/ProcessUtils.java @@ -47,7 +47,7 @@ public class ProcessUtils { private final InputStream is; private final StringBuffer sink = new StringBuffer(); - StreamGobbler(InputStream is, String name) { + protected StreamGobbler(InputStream is, String name) { this.is = is; setName(name); } diff --git a/commons/src/main/java/org/archive/util/PropertyUtils.java b/commons/src/main/java/org/archive/util/PropertyUtils.java index 955e772f..083615f6 100644 --- a/commons/src/main/java/org/archive/util/PropertyUtils.java +++ b/commons/src/main/java/org/archive/util/PropertyUtils.java @@ -74,7 +74,7 @@ public class PropertyUtils { return interpolateWithProperties(original,System.getProperties()); } - static String propRefPattern = "\\$\\{([^{}]+)\\}"; + protected static String propRefPattern = "\\$\\{([^{}]+)\\}"; /** * Given a string which may contain expressions of the form diff --git a/commons/src/main/java/org/archive/util/Recorder.java b/commons/src/main/java/org/archive/util/Recorder.java index c07a04c2..298481ea 100644 --- a/commons/src/main/java/org/archive/util/Recorder.java +++ b/commons/src/main/java/org/archive/util/Recorder.java @@ -293,7 +293,7 @@ public class Recorder { } - static ThreadLocal currentRecorder = new ThreadLocal(); + protected static ThreadLocal currentRecorder = new ThreadLocal(); public static void setHttpRecorder(Recorder httpRecorder) { currentRecorder.set(httpRecorder); @@ -331,7 +331,7 @@ public class Recorder { this.inputIsChunked = chunked; } - static Set SUPPORTED_ENCODINGS = new HashSet(); + protected static Set SUPPORTED_ENCODINGS = new HashSet(); static { SUPPORTED_ENCODINGS.add("gzip"); SUPPORTED_ENCODINGS.add("x-gzip"); diff --git a/commons/src/main/java/org/archive/util/SURT.java b/commons/src/main/java/org/archive/util/SURT.java index c924637b..e5797935 100644 --- a/commons/src/main/java/org/archive/util/SURT.java +++ b/commons/src/main/java/org/archive/util/SURT.java @@ -61,10 +61,10 @@ import org.archive.net.UURIFactory; * @author gojomo */ public class SURT { - static char DOT = '.'; - static String BEGIN_TRANSFORMED_AUTHORITY = "("; - static String TRANSFORMED_HOST_DELIM = ","; - static String END_TRANSFORMED_AUTHORITY = ")"; + protected static char DOT = '.'; + protected static String BEGIN_TRANSFORMED_AUTHORITY = "("; + protected static String TRANSFORMED_HOST_DELIM = ","; + protected static String END_TRANSFORMED_AUTHORITY = ")"; // 1: scheme:// // 2: userinfo (if present) @@ -73,7 +73,7 @@ public class SURT { // 5: other host // 6: :port // 7: path - static String URI_SPLITTER = + protected static String URI_SPLITTER = "^(\\w+://)(?:([-\\w\\.!~\\*'\\(\\)%;:&=+$,]+?)(@))?"+ // 1 2 3 "(?:((?:\\d{1,3}\\.){3}\\d{1,3})|(\\S+?))(:\\d+)?(/\\S*)?$"; diff --git a/commons/src/main/java/org/archive/util/Supplier.java b/commons/src/main/java/org/archive/util/Supplier.java index 16a68ae6..0e629dea 100644 --- a/commons/src/main/java/org/archive/util/Supplier.java +++ b/commons/src/main/java/org/archive/util/Supplier.java @@ -26,7 +26,7 @@ package org.archive.util; * @param */ public class Supplier { - V instance; + protected V instance; public Supplier() { super(); diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index 1238517f..8691333a 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -88,7 +88,7 @@ public class UriUtils { // naive likely-uri test: // no whitespace or '<' or '>' // at least one '.' or '/'; - static final String NAIVE_LIKELY_URI_PATTERN = "[^<>\\s]*[\\./][^<>\\s]*"; + protected static final String NAIVE_LIKELY_URI_PATTERN = "[^<>\\s]*[\\./][^<>\\s]*"; public static boolean isPossibleUri(CharSequence candidate) { return TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate); @@ -350,7 +350,7 @@ public class UriUtils { // determines whether a string is likely URI // (no whitespace or '<' '>', has an internal dot or some slash, // begins and ends with either '/' or a word-char) - static final String STRING_URI_DETECTOR = + protected static final String STRING_URI_DETECTOR = "(?:\\w|[\\.]{0,2}/)[\\S&&[^<>]]*(?:\\.|/)[\\S&&[^<>]]*(?:\\w|/)"; @@ -382,7 +382,7 @@ public class UriUtils { // without requiring quotes -- this can indicate whether // an HTML tag attribute that isn't definitionally a // URI might be one anyway, as in form-tag VALUE attributes - static final String LIKELY_URI_PATH = + protected static final String LIKELY_URI_PATH = "(\\.{0,2}[^\\.\\n\\r\\s\"']*(\\.[^\\.\\n\\r\\s\"']+)+)"; public static boolean isLikelyUriHtmlContextLegacy(CharSequence candidate) { diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java index 70984edb..cfabb1dc 100644 --- a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java +++ b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java @@ -61,9 +61,9 @@ public class ANVLRecord extends ArrayList implements UTF8Bytes { * An ANVL 'newline'. * @see http://en.wikipedia.org/wiki/CRLF */ - static final String CRLF = "\r\n"; + protected static final String CRLF = "\r\n"; - static final String FOLD_PREFIX = CRLF + ' '; + protected static final String FOLD_PREFIX = CRLF + ' '; public ANVLRecord() { super(); diff --git a/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java b/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java index f9c974ba..380ed948 100644 --- a/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java +++ b/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java @@ -35,8 +35,8 @@ import com.sleepycat.je.EnvironmentConfig; * @author gojomo */ public class EnhancedEnvironment extends Environment { - StoredClassCatalog classCatalog; - Database classCatalogDB; + protected StoredClassCatalog classCatalog; + protected Database classCatalogDB; /** * Constructor diff --git a/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java b/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java index ceb664b8..fabb0b94 100644 --- a/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java +++ b/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java @@ -28,9 +28,9 @@ public class ArrayLongFPCache implements LongFPSet { public static final int DEFAULT_CAPACITY = 1 << 20; // 1 million, 8MB public static final int DEFAULT_SMEAR = 5; - long cache[] = new long[DEFAULT_CAPACITY]; - int smear = DEFAULT_SMEAR; - int count = 0; + protected long cache[] = new long[DEFAULT_CAPACITY]; + protected int smear = DEFAULT_SMEAR; + protected int count = 0; public void setCapacity(int newCapacity) { long[] oldCache = cache; diff --git a/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java b/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java index bb3fdb76..0b0b19d1 100644 --- a/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java +++ b/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java @@ -47,7 +47,7 @@ abstract public class LongFPSetTestCase extends TestCase { fpSet = makeLongFPSet(); } - abstract LongFPSet makeLongFPSet(); + protected abstract LongFPSet makeLongFPSet(); /** check that we can add fingerprints */ public void testAdd() { diff --git a/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java b/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java index 2d6fee29..c123c2bc 100644 --- a/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java +++ b/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java @@ -28,9 +28,9 @@ import java.util.NoSuchElementException; * @author gojomo */ public class CompositeIterator implements Iterator { - ArrayList> iterators = new ArrayList>(); - Iterator currentIterator; - int indexOfCurrentIterator = -1; + protected ArrayList> iterators = new ArrayList>(); + protected Iterator currentIterator; + protected int indexOfCurrentIterator = -1; /** * Moves to the next (non empty) iterator. Returns false if there are no diff --git a/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java b/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java index fe1a9869..850472bd 100644 --- a/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java +++ b/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java @@ -179,7 +179,7 @@ public class DefaultBlockFileSystem implements BlockFileSystem { * @return that entry, or null if no such entry exists * @throws IOException if an IO error occurs */ - Entry getEntry(int entryNumber) throws IOException { + protected Entry getEntry(int entryNumber) throws IOException { // Entry numbers < 0 typically indicate an end-of-stream. if (entryNumber < 0) { return null; diff --git a/commons/src/main/java/org/archive/util/ms/PieceTable.java b/commons/src/main/java/org/archive/util/ms/PieceTable.java index 7a95c4f6..dd1401a6 100644 --- a/commons/src/main/java/org/archive/util/ms/PieceTable.java +++ b/commons/src/main/java/org/archive/util/ms/PieceTable.java @@ -70,14 +70,14 @@ import org.archive.io.SeekInputStream; */ class PieceTable { - final static Logger LOGGER + private final static Logger LOGGER = Logger.getLogger(PieceTable.class.getName()); /** The bit that indicates if a piece uses Cp1252 or unicode. */ - final static int CP1252_INDICATOR = 1 << 30; + protected final static int CP1252_INDICATOR = 1 << 30; /** The mask to use to clear the Cp1252 flag bit. */ - final static int CP1252_MASK = ~(3 << 30); + protected final static int CP1252_MASK = ~(3 << 30); /** The total number of pieces in the table. */ private int count; diff --git a/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java b/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java index f4b479af..42e9ea18 100644 --- a/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java +++ b/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java @@ -101,7 +101,7 @@ class OpenJDK7InflaterInputStream extends FilterInputStream { // IA RENAMED CLAS this(in, inf, 512); } - boolean usesDefaultInflater = false; + protected boolean usesDefaultInflater = false; /** * Creates a new input stream with a default decompressor and buffer size. diff --git a/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java b/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java index b952b00f..3e7cd9d2 100644 --- a/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java +++ b/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java @@ -55,7 +55,7 @@ public class LongFPSetCacheTest extends LongFPSetTestCase { return new TestSuite(LongFPSetCacheTest.class); } - LongFPSet makeLongFPSet() { + protected LongFPSet makeLongFPSet() { return new LongFPSetCache(); } diff --git a/commons/src/test/java/org/archive/util/fingerprint/MemLongFPSetTest.java b/commons/src/test/java/org/archive/util/fingerprint/MemLongFPSetTest.java index 215055bd..5b955f27 100644 --- a/commons/src/test/java/org/archive/util/fingerprint/MemLongFPSetTest.java +++ b/commons/src/test/java/org/archive/util/fingerprint/MemLongFPSetTest.java @@ -56,7 +56,7 @@ public class MemLongFPSetTest extends LongFPSetTestCase { return new TestSuite(MemLongFPSetTest.class); } - LongFPSet makeLongFPSet() { + protected LongFPSet makeLongFPSet() { return new MemLongFPSet(); } diff --git a/engine/src/main/java/org/archive/crawler/event/StatSnapshotEvent.java b/engine/src/main/java/org/archive/crawler/event/StatSnapshotEvent.java index 1febce1f..efd09e52 100644 --- a/engine/src/main/java/org/archive/crawler/event/StatSnapshotEvent.java +++ b/engine/src/main/java/org/archive/crawler/event/StatSnapshotEvent.java @@ -33,7 +33,7 @@ import org.springframework.context.ApplicationEvent; */ public class StatSnapshotEvent extends ApplicationEvent { private static final long serialVersionUID = 1L; - CrawlStatSnapshot snapshot; + protected CrawlStatSnapshot snapshot; public StatSnapshotEvent(StatisticsTracker stats, CrawlStatSnapshot snapshot) { super(stats); diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index 56fe41dc..305137c2 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -82,7 +82,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn final private static Logger LOGGER = Logger.getLogger(ActionDirectory.class.getName()); - ScheduledExecutorService executor; + protected ScheduledExecutorService executor; /** how long after crawl start to first scan action directory */ protected int initialDelaySeconds = 10; @@ -119,7 +119,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn this.doneDir = doneDir; } - ApplicationContext appCtx; + protected ApplicationContext appCtx; public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.appCtx = applicationContext; } @@ -278,7 +278,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn /** shared ScriptEngineManager */ - static ScriptEngineManager MANAGER = new ScriptEngineManager(); + protected static ScriptEngineManager MANAGER = new ScriptEngineManager(); /** * Try the actionFile as a script, deducing the proper scripting diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 3b5d8b2e..0e939bca 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -67,9 +67,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha /** Next overall series checkpoint number */ protected int nextCheckpointNumber = 1; - Checkpoint checkpointInProgress; + protected Checkpoint checkpointInProgress; - CrawlStatSnapshot lastCheckpointSnapshot = null; + protected CrawlStatSnapshot lastCheckpointSnapshot = null; /** service for auto-checkpoint tasks at an interval */ protected Timer timer = new Timer(true); @@ -90,7 +90,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha * Period at which to create automatic checkpoints; -1 means * no auto checkpointing. */ - long checkpointIntervalMinutes = -1; + protected long checkpointIntervalMinutes = -1; public long getCheckpointIntervalMinutes() { return checkpointIntervalMinutes; } @@ -102,7 +102,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } } - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; @Autowired(required=false) public void setRecoveryCheckpoint(Checkpoint checkpoint) { this.recoveryCheckpoint = checkpoint; @@ -122,7 +122,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } // ApplicationContextAware implementation, for eventing - AbstractApplicationContext appCtx; + protected AbstractApplicationContext appCtx; public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.appCtx = (AbstractApplicationContext)applicationContext; } @@ -186,7 +186,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha periodMs + " milliseconds."); } - boolean isRunning = false; + protected boolean isRunning = false; public synchronized boolean isRunning() { return isRunning; } @@ -346,7 +346,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } } - static Validator VALIDATOR = new CheckpointValidator(); + protected static Validator VALIDATOR = new CheckpointValidator(); @Override public Validator getValidator() { return VALIDATOR; diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java index a2e22de9..e6e7c052 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointSuccessEvent.java @@ -30,7 +30,7 @@ import org.springframework.context.ApplicationEvent; */ public class CheckpointSuccessEvent extends ApplicationEvent { private static final long serialVersionUID = 1L; - Checkpoint checkpoint; + protected Checkpoint checkpoint; public Checkpoint getCheckpoint() { return checkpoint; diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java index b2c1d543..81504977 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -72,12 +72,12 @@ implements Serializable, private static final long serialVersionUID = 1L; // ApplicationContextAware implementation, for eventing - AbstractApplicationContext appCtx; + protected AbstractApplicationContext appCtx; public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.appCtx = (AbstractApplicationContext)applicationContext; } - CrawlMetadata metadata; + protected CrawlMetadata metadata; public CrawlMetadata getMetadata() { return metadata; } @@ -181,7 +181,7 @@ implements Serializable, /** * Maximum number of threads processing URIs at the same time. */ - int maxToeThreads; + protected int maxToeThreads; public int getMaxToeThreads() { return maxToeThreads; } @@ -194,7 +194,7 @@ implements Serializable, } /** whether to keep running (without pause or finish) when frontier is empty */ - boolean runWhileEmpty = false; + protected boolean runWhileEmpty = false; public boolean getRunWhileEmpty() { return runWhileEmpty; } @@ -203,7 +203,7 @@ implements Serializable, } /** whether to pause at crawl start */ - boolean pauseAtStart = true; + protected boolean pauseAtStart = true; public boolean getPauseAtStart() { return pauseAtStart; } @@ -215,7 +215,7 @@ implements Serializable, * Size in bytes of in-memory buffer to record outbound traffic. One such * buffer is reserved for every ToeThread. */ - int recorderOutBufferBytes = 16 * 1024; // 16KiB + protected int recorderOutBufferBytes = 16 * 1024; // 16KiB public int getRecorderOutBufferBytes() { return recorderOutBufferBytes; } @@ -227,7 +227,7 @@ implements Serializable, * Size in bytes of in-memory buffer to record inbound traffic. One such * buffer is reserved for every ToeThread. */ - int recorderInBufferBytes = 512 * 1024; // 512KiB + protected int recorderInBufferBytes = 512 * 1024; // 512KiB public int getRecorderInBufferBytes() { return recorderInBufferBytes; } @@ -274,7 +274,7 @@ implements Serializable, public CrawlController() { } - transient AlertThreadGroup alertThreadGroup; + transient protected AlertThreadGroup alertThreadGroup; public void start() { // cache AlertThreadGroup for later ToePool launch @@ -300,7 +300,7 @@ implements Serializable, isRunning = true; } - boolean isRunning = false; + protected boolean isRunning = false; public boolean isRunning() { return isRunning; } @@ -328,13 +328,13 @@ implements Serializable, // TODO: provide better knowledge/guard against twice-starting - boolean hasStarted = false; + protected boolean hasStarted = false; public boolean hasStarted() { return hasStarted; } - boolean isStopComplete = false; + protected boolean isStopComplete = false; public boolean isStopComplete() { return isStopComplete; } @@ -666,7 +666,7 @@ implements Serializable, public void startCheckpoint(Checkpoint checkpointInProgress) {} public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException {} public void finishCheckpoint(Checkpoint checkpointInProgress) {} - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { this.recoveryCheckpoint = recoveryCheckpoint; } diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 124b6f7a..230c48e2 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -85,15 +85,15 @@ public class CrawlJob implements Comparable, ApplicationListener> getInactiveQueuesByPrecedence() { + protected SortedMap> getInactiveQueuesByPrecedence() { return inactiveQueuesByPrecedence; } @@ -111,7 +111,7 @@ implements Checkpointable, BeanNameAware { * @see org.archive.crawler.frontier.WorkQueueFrontier#getRetiredQueues() */ @Override - Queue getRetiredQueues() { + protected Queue getRetiredQueues() { return retiredQueues; } @@ -276,7 +276,7 @@ implements Checkpointable, BeanNameAware { dispositionInProgressLock.writeLock().unlock(); } - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; @Autowired(required=false) public void setRecoveryCheckpoint(Checkpoint checkpoint) { this.recoveryCheckpoint = checkpoint; @@ -357,14 +357,14 @@ implements Checkpointable, BeanNameAware { * @see org.archive.crawler.frontier.WorkQueueFrontier#createInactiveQueueForPrecedence(int) */ @Override - Queue createInactiveQueueForPrecedence(int precedence) { + protected Queue createInactiveQueueForPrecedence(int precedence) { return createInactiveQueueForPrecedence(precedence, false); } /** * Optionally reuse prior data, for use when resuming from a checkpoint */ - Queue createInactiveQueueForPrecedence(int precedence, boolean usePriorData) { + protected Queue createInactiveQueueForPrecedence(int precedence, boolean usePriorData) { return bdb.getStoredQueue("inactiveQueues-"+precedence, String.class, usePriorData); } diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index c1846c80..e38b525d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -386,7 +386,7 @@ public class BdbMultipleWorkQueues { * @param classKey String key to derive origin byte key from * @return a byte array key */ - static byte[] calculateOriginKey(String classKey) { + protected static byte[] calculateOriginKey(String classKey) { byte[] classKeyBytes = null; int len = 0; try { @@ -422,7 +422,7 @@ public class BdbMultipleWorkQueues { * @param curi * @return a DatabaseEntry key for the CrawlURI */ - static DatabaseEntry calculateInsertKey(CrawlURI curi) { + protected static DatabaseEntry calculateInsertKey(CrawlURI curi) { byte[] classKeyBytes = null; int len = 0; classKeyBytes = curi.getClassKey().getBytes(Charsets.UTF_8); @@ -441,7 +441,7 @@ public class BdbMultipleWorkQueues { } - static String insertKeyToString(DatabaseEntry holderKey) { + protected static String insertKeyToString(DatabaseEntry holderKey) { StringBuilder result = new StringBuilder(); byte[] data = holderKey.getData(); int p = findFirstZero(data); @@ -503,7 +503,7 @@ public class BdbMultipleWorkQueues { * need access. * @see Deferred Write Databases */ - void sync() { + protected void sync() { if (this.pendingUrisDB == null) { return; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java index 6e95d39d..6f900119 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java @@ -33,7 +33,7 @@ import org.archive.spring.KeyedProperties; public abstract class QueueAssignmentPolicy implements Serializable, HasKeyedProperties { private static final long serialVersionUID = 1L; - KeyedProperties kp = new KeyedProperties(); + protected KeyedProperties kp = new KeyedProperties(); public KeyedProperties getKeyedProperties() { return kp; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java b/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java index 69a5433a..82d1bd60 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java +++ b/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java @@ -40,7 +40,7 @@ public class RecyclingSerialBinding extends SerialBinding { /** * Thread-local cache of reusable FastOutputStream */ - ThreadLocal fastOutputStreamHolder + protected ThreadLocal fastOutputStreamHolder = new ThreadLocal(); private ClassCatalog classCatalog; diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 2b2f4ea2..fc0dc8bc 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -38,7 +38,7 @@ implements //for when neat class-key fails us protected static String DEFAULT_CLASS_KEY = "default..."; - LongToIntConsistentHash conhash = new LongToIntConsistentHash(); + protected LongToIntConsistentHash conhash = new LongToIntConsistentHash(); /** * Whether to always defer to a previously-assigned key inside diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java index 3e8259f1..5ce4488b 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -211,7 +211,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, * * @param balance to use */ - void setSessionBudget(int budget) { + protected void setSessionBudget(int budget) { this.sessionBudget = budget; } @@ -248,7 +248,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, * * @param budget */ - void setTotalBudget(long budget) { + protected void setTotalBudget(long budget) { this.totalBudget = budget; } @@ -609,7 +609,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, * * @param b new value for retired status */ - void setRetired(boolean b) { + protected void setRetired(boolean b) { this.retired = b; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index b7db0ced..6dc67082 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -98,7 +98,7 @@ implements Closeable, * will be "long snoozed" instead of "short snoozed". A "long snoozed" * queue may be swapped to disk because it's not needed soon. */ - long snoozeLongMs = 5L*60L*1000L; + protected long snoozeLongMs = 5L*60L*1000L; public long getSnoozeLongMs() { return snoozeLongMs; } @@ -110,7 +110,7 @@ implements Closeable, Logger.getLogger(WorkQueueFrontier.class.getName()); // ApplicationContextAware implementation, for eventing - AbstractApplicationContext appCtx; + protected AbstractApplicationContext appCtx; public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.appCtx = (AbstractApplicationContext)applicationContext; } @@ -482,14 +482,14 @@ implements Closeable, * Return a sorted map of all queues of WorkQueue keys, keyed by precedence * @return SortedMap> of inactiveQueues */ - abstract SortedMap> getInactiveQueuesByPrecedence(); + protected abstract SortedMap> getInactiveQueuesByPrecedence(); /** * Create an inactiveQueue to hold queue names at the given precedence * @param precedence * @return Queue for names of inactive queues */ - abstract Queue createInactiveQueueForPrecedence(int precedence); + protected abstract Queue createInactiveQueueForPrecedence(int precedence); /** * Put the given queue on the retiredQueues queue @@ -513,7 +513,7 @@ implements Closeable, * * @return Queue of retired queue names */ - abstract Queue getRetiredQueues(); + protected abstract Queue getRetiredQueues(); /** * Accommodate any changes in retirement-determining settings (like diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java index 902e9bec..b0694e17 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java @@ -25,7 +25,7 @@ package org.archive.crawler.frontier.precedence; public class SimplePrecedenceProvider extends PrecedenceProvider { private static final long serialVersionUID = 3132028374638763487L; - int precedence; + protected int precedence; public SimplePrecedenceProvider(int p) { this.precedence = p; diff --git a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java index 5f511b48..4d338130 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java +++ b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java @@ -61,7 +61,7 @@ public class MigrateH1to3Tool { protected Document sourceOrderXmlDom; - static DocumentBuilder DOCUMENT_BUILDER; + protected static DocumentBuilder DOCUMENT_BUILDER; static { try { diff --git a/engine/src/main/java/org/archive/crawler/monitor/DiskSpaceMonitor.java b/engine/src/main/java/org/archive/crawler/monitor/DiskSpaceMonitor.java index e6ebb871..7afa3e12 100644 --- a/engine/src/main/java/org/archive/crawler/monitor/DiskSpaceMonitor.java +++ b/engine/src/main/java/org/archive/crawler/monitor/DiskSpaceMonitor.java @@ -34,11 +34,11 @@ import org.springframework.context.ApplicationListener; public class DiskSpaceMonitor implements ApplicationListener { private static final Logger logger = Logger.getLogger(DiskSpaceMonitor.class.getName()); - List monitorPaths = new ArrayList(); - long pauseThresholdMiB = 500; - CrawlController controller; - ConfigPathConfigurer configPathConfigurer; - boolean monitorConfigPaths = true; + protected List monitorPaths = new ArrayList(); + protected long pauseThresholdMiB = 500; + protected CrawlController controller; + protected ConfigPathConfigurer configPathConfigurer; + protected boolean monitorConfigPaths = true; /** * @param monitorPaths List of filesystem paths that should be monitored for available space. diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 01879ac0..0cba02c6 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -151,7 +151,7 @@ public class DispositionProcessor extends Processor { * Auto-discovered module providing configured (or overridden) * User-Agent value and RobotsHonoringPolicy */ - CrawlMetadata metadata; + protected CrawlMetadata metadata; public CrawlMetadata getMetadata() { return metadata; } diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java index 54af2468..4143a581 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java @@ -67,7 +67,7 @@ public class LowDiskPauseProcessor extends Processor { * List of filessystem mounts whose 'available' space should be monitored * via 'df' (if available). */ - List monitorMounts = new ArrayList(); + protected List monitorMounts = new ArrayList(); public List getMonitorMounts() { return this.monitorMounts; } @@ -79,7 +79,7 @@ public class LowDiskPauseProcessor extends Processor { * When available space on any monitored mounts falls below this threshold, * the crawl will be paused. */ - int pauseThresholdKb = 500*1024; // 500MB + protected int pauseThresholdKb = 500*1024; // 500MB public int getPauseThresholdKb() { return this.pauseThresholdKb; } @@ -91,7 +91,7 @@ public class LowDiskPauseProcessor extends Processor { * Available space via 'df' is rechecked after every increment of this much * content (uncompressed) is observed. */ - int recheckThresholdKb = 200*1024; // 200MB + protected int recheckThresholdKb = 200*1024; // 200MB public int getRecheckThresholdKb() { return this.recheckThresholdKb; } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 1c2c7f19..8df658c0 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -104,7 +104,7 @@ public class PreconditionEnforcer extends Processor { * Auto-discovered module providing configured (or overridden) * User-Agent value and RobotsHonoringPolicy */ - CrawlMetadata metadata; + protected CrawlMetadata metadata; public CrawlMetadata getMetadata() { return metadata; } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java index 9e356e87..ff5c307a 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java @@ -99,7 +99,7 @@ public class RuntimeLimitEnforcer extends Processor { * The amount of time, in seconds, that the crawl will be allowed to run * before this processor performs it's 'end operation.' */ - long runtimeSeconds = 24*60*60L; // 1 day + protected long runtimeSeconds = 24*60*60L; // 1 day public long getRuntimeSeconds() { return this.runtimeSeconds; } @@ -122,7 +122,7 @@ public class RuntimeLimitEnforcer extends Processor { * processor) fetch status code. This will cause all the URIs queued to wind * up in the crawl.log. */ - Operation expirationOperation = Operation.PAUSE; + protected Operation expirationOperation = Operation.PAUSE; public Operation getExpirationOperation() { return this.expirationOperation; } diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java index 53296cd1..9e56689e 100644 --- a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -84,7 +84,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { * Whether to apply the mapping to a URI being processed itself, for example * early in processing (while its status is still 'unattempted'). */ - boolean checkUri = true; + protected boolean checkUri = true; public boolean getCheckUri() { return this.checkUri; } @@ -96,7 +96,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { * Whether to apply the mapping to discovered outlinks, for example after * extraction has occurred. */ - boolean checkOutlinks = true; + protected boolean checkOutlinks = true; public boolean getCheckOutlinks() { return this.checkOutlinks; } @@ -107,7 +107,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { /** * Decide rules to determine if an outlink is subject to mapping. */ - DecideRule outlinkRule = new AcceptDecideRule(); + protected DecideRule outlinkRule = new AcceptDecideRule(); public DecideRule getOutlinkRule() { return this.outlinkRule; } @@ -119,7 +119,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { * Name of local crawler node; mappings to this name result in normal * processing (no diversion). */ - String localName = "."; + protected String localName = "."; public String getLocalName() { return this.localName; } @@ -130,7 +130,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { /** * Directory to write diversion logs. */ - ConfigPath diversionDir = + protected ConfigPath diversionDir = new ConfigPath("diverted URIs subdirectory","diversions"); public ConfigPath getDiversionDir() { return this.diversionDir; @@ -145,7 +145,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { * rotation). * */ - int rotationDigits = 10; + protected int rotationDigits = 10; public int getRotationDigits() { return this.rotationDigits; } @@ -156,7 +156,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { /** * Mapping of target crawlers to logs (PrintWriters) */ - HashMap diversionLogs + protected HashMap diversionLogs = new HashMap(); /** @@ -164,7 +164,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { * current time doesn't match, it's time to close all * current logs. */ - String logGeneration = ""; + protected String logGeneration = ""; protected ArrayLongFPCache cache; diff --git a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java index 21c717c4..22016b9c 100644 --- a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java @@ -52,7 +52,7 @@ public class HashCrawlMapper extends CrawlMapper { * Number of crawlers among which to split up the URIs. Their names are * assumed to be 0..N-1. */ - long crawlerCount = 1L; + protected long crawlerCount = 1L; public long getCrawlerCount() { return this.crawlerCount; } diff --git a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java index 85c1c371..96358ae7 100644 --- a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java @@ -86,7 +86,7 @@ public class LexicalCrawlMapper extends CrawlMapper { * range, the second the crawler node to which URIs in the key range should * be mapped. This setting is ignored if MAP_URI is specified. */ - ConfigPath mapPath = new ConfigPath("map specification file","lexicalcrawlmapper.config"); + protected ConfigPath mapPath = new ConfigPath("map specification file","lexicalcrawlmapper.config"); public ConfigPath getMapPath() { return this.mapPath; } @@ -102,7 +102,7 @@ public class LexicalCrawlMapper extends CrawlMapper { * be mapped. This setting takes precedence over MAP_PATH; if both are * specified, then MAP_PATH is ignored. */ - String mapUri = ""; + protected String mapUri = ""; public String getMapUri() { return this.mapUri; } @@ -123,7 +123,7 @@ public class LexicalCrawlMapper extends CrawlMapper { * Mapping of classKey ranges (as represented by their start) to * crawlers (by abstract name/filename) */ - TreeMap map = new TreeMap(); + protected TreeMap map = new TreeMap(); /** * Constructor. diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 8852757c..0f388199 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -76,7 +76,7 @@ public class CrawlerLoggerModule * Whether to include the "extra info" field for each entry in crawl.log. * "Extra info" is arbitrary JSON. It is the last field of the log line. */ - boolean logExtraInfo = false; + protected boolean logExtraInfo = false; public boolean getLogExtraInfo() { return logExtraInfo; } @@ -221,7 +221,7 @@ public class CrawlerLoggerModule isRunning = true; } - boolean isRunning = false; + protected boolean isRunning = false; public boolean isRunning() { return this.isRunning; } @@ -386,7 +386,7 @@ public class CrawlerLoggerModule public void finishCheckpoint(Checkpoint checkpointInProgress) {} - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; @Autowired(required=false) public void setRecoveryCheckpoint(Checkpoint checkpoint) { this.recoveryCheckpoint = checkpoint; diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 6ffd8ef2..c7b4e376 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -180,7 +180,7 @@ public class StatisticsTracker this.liveHostReportSize = liveHostReportSize; } - ApplicationContext appCtx; + protected ApplicationContext appCtx; public void setApplicationContext(ApplicationContext appCtx) throws BeansException { this.appCtx = appCtx; } @@ -195,7 +195,7 @@ public class StatisticsTracker * Whether to maintain seed disposition records (expensive in * crawls with millions of seeds) */ - boolean trackSeeds = true; + protected boolean trackSeeds = true; public boolean getTrackSeeds() { return this.trackSeeds; } @@ -208,7 +208,7 @@ public class StatisticsTracker * crawls with large numbers of source-tags (seeds) or large crawls * over many hosts */ - boolean trackSources = true; + protected boolean trackSources = true; public boolean getTrackSources() { return this.trackSources; } @@ -219,7 +219,7 @@ public class StatisticsTracker /** * The interval between writing progress information to log. */ - int intervalSeconds = 20; + protected int intervalSeconds = 20; public int getIntervalSeconds() { return this.intervalSeconds; } @@ -231,7 +231,7 @@ public class StatisticsTracker * Number of crawl-stat sample snapshots to keep for calculation * purposes. */ - int keepSnapshotsCount = 5; + protected int keepSnapshotsCount = 5; public int getKeepSnapshotsCount() { return this.keepSnapshotsCount; } @@ -249,24 +249,24 @@ public class StatisticsTracker } /** wall-clock time the crawl started */ - long crawlStartTime; + protected long crawlStartTime; /** wall-clock time the crawl ended */ - long crawlEndTime = -1; // Until crawl ends, this value is -1. + protected long crawlEndTime = -1; // Until crawl ends, this value is -1. /** wall-clock time of last pause, while pause in progres */ - long crawlPauseStarted = 0; + protected long crawlPauseStarted = 0; /** duration tally of all time spent in paused state */ - long crawlTotalPausedTime = 0; + protected long crawlTotalPausedTime = 0; /** snapshots of crawl tallies and rates */ - LinkedList snapshots = new LinkedList(); + protected LinkedList snapshots = new LinkedList(); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + protected ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); /* * Cumulative data */ /** tally sizes novel, verified (same hash), vouched (not-modified) */ - CrawledBytesHistotable crawledBytes = new CrawledBytesHistotable(); + protected CrawledBytesHistotable crawledBytes = new CrawledBytesHistotable(); public CrawledBytesHistotable getCrawledBytes() { return crawledBytes; @@ -298,8 +298,8 @@ public class StatisticsTracker */ protected ObjectIdentityCache processedSeedsRecords = new ObjectIdentityMemCache(); - long seedsTotal = -1; - long seedsCrawled = -1; + protected long seedsTotal = -1; + protected long seedsCrawled = -1; public StatisticsTracker() { @@ -330,7 +330,7 @@ public class StatisticsTracker this.reports = reports; } - boolean isRunning = false; + protected boolean isRunning = false; public boolean isRunning() { return isRunning; } @@ -1053,7 +1053,7 @@ public class StatisticsTracker } // BeanNameAware - String beanName; + protected String beanName; public void setBeanName(String name) { this.beanName = name; } @@ -1093,7 +1093,7 @@ public class StatisticsTracker } } public void finishCheckpoint(Checkpoint checkpointInProgress) {} - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { this.recoveryCheckpoint = recoveryCheckpoint; } diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index 090f3160..3dd4fcc0 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -55,8 +55,8 @@ import org.springframework.beans.BeansException; * @contributor nlevitt */ public class BeanBrowseResource extends JobRelatedResource { - PathSharingContext appCtx; - String beanPath; + protected PathSharingContext appCtx; + protected String beanPath; public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException { super(ctx, req, res); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index 449a1103..aa6322dc 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -40,8 +40,8 @@ import org.restlet.resource.FileRepresentation; * @contributor gojomo */ public class EditRepresentation extends CharacterRepresentation { - FileRepresentation fileRepresentation; - EnhDirectoryResource dirResource; + protected FileRepresentation fileRepresentation; + protected EnhDirectoryResource dirResource; public EditRepresentation(FileRepresentation representation, EnhDirectoryResource resource) { super(MediaType.TEXT_HTML); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java index 1a6d27f9..33dea5b8 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java @@ -48,7 +48,7 @@ import org.restlet.util.Template; * @contributor gojomo */ public class EngineApplication extends Application { - Engine engine; + protected Engine engine; public EngineApplication(Engine engine) { this.engine = engine; getMetadataService().addExtension("log", MediaType.TEXT_PLAIN ); @@ -76,7 +76,7 @@ public class EngineApplication extends Application { getContext(), engine.getJobsDir().toURI().toString() /*TODO: changeme*/) { @Override - Reference determineRootRef(Request request) { + protected Reference determineRootRef(Request request) { String ref = "file:/"; return new Reference(ref); }}; @@ -90,7 +90,7 @@ public class EngineApplication extends Application { getContext(), engine.getJobsDir().toURI().toString() /*TODO: changeme*/) { @Override - Reference determineRootRef(Request request) { + protected Reference determineRootRef(Request request) { try { return new Reference( EngineApplication.this.getEngine() diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java index b816feed..d1ad56ce 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java @@ -40,9 +40,9 @@ import org.restlet.data.Response; * @contributor gojomo */ public abstract class EnhDirectory extends org.restlet.Directory { - IOFileFilter editFilter = FileFilterUtils.falseFileFilter(); - IOFileFilter pageFilter = FileFilterUtils.falseFileFilter(); - IOFileFilter tailFilter = FileFilterUtils.falseFileFilter(); + protected IOFileFilter editFilter = FileFilterUtils.falseFileFilter(); + protected IOFileFilter pageFilter = FileFilterUtils.falseFileFilter(); + protected IOFileFilter tailFilter = FileFilterUtils.falseFileFilter(); public EnhDirectory(Context context, Reference rootLocalReference) { super(context, rootLocalReference); @@ -72,7 +72,7 @@ public abstract class EnhDirectory extends org.restlet.Directory { return retVal; } - abstract Reference determineRootRef(Request request); + protected abstract Reference determineRootRef(Request request); public boolean allowsEdit(File file) { return editFilter.accept(file); diff --git a/engine/src/main/java/org/archive/crawler/restlet/Flash.java b/engine/src/main/java/org/archive/crawler/restlet/Flash.java index 93bf78a8..989d66d6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/Flash.java +++ b/engine/src/main/java/org/archive/crawler/restlet/Flash.java @@ -46,8 +46,8 @@ public class Flash { /** usual types */ public enum Kind {ACK, NACK, ADVISORY} - static long nextdrop = RandomUtils.nextLong(); - static Map dropboxes = new LinkedHashMap() { + protected static long nextdrop = RandomUtils.nextLong(); + protected static Map dropboxes = new LinkedHashMap() { private static final long serialVersionUID = 1L; @Override protected boolean removeEldestEntry(Entry eldest) { diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index fc65905e..aa1bab8c 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -55,9 +55,9 @@ import org.springframework.beans.InvalidPropertyException; * @contributor nlevitt */ public abstract class JobRelatedResource extends BaseResource { - CrawlJob cj; + protected CrawlJob cj; - IdentityHashMap beanToNameMap; + protected IdentityHashMap beanToNameMap; public JobRelatedResource(Context ctx, Request req, Response res) throws ResourceException { super(ctx, req, res); @@ -517,7 +517,7 @@ public abstract class JobRelatedResource extends BaseResource { } /** suppress problematic properties */ - static HashSet HIDDEN_PROPS = new HashSet( + protected static HashSet HIDDEN_PROPS = new HashSet( Arrays.asList(new String[] {"class","declaringClass","keyedProperties","running","first","last","empty", "inbound", "outbound", "cookiesMap"} )); diff --git a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java index 1f8c3c0c..ad7e4fc7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java @@ -50,24 +50,24 @@ import org.restlet.resource.FileRepresentation; public class PagedRepresentation extends CharacterRepresentation { // passed-in at construction /** wrapped FileRepresentation **/ - FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; /** wrapped EnhDirectoryResource; used to formulate self-links **/ - EnhDirectoryResource dirResource; + protected EnhDirectoryResource dirResource; /** position in file around which to fetch lines **/ - long position; + protected long position; /** desired line count; negative to go back from position; default 128 **/ - int lineCount; + protected int lineCount; /** whether to display lines in reversed order (latest first) **/ - boolean reversedOrder; + protected boolean reversedOrder; // created when file is scanned /** text lines **/ - List lines; + protected List lines; /** position range [start-of-first-line, past-end-of-last-line] in file **/ - LongRange range; + protected LongRange range; /** File **/ - File file; + protected File file; // TODO: maybe, freeze length for more consistent display of growing files // (now, as length/%/bumper are written after lines retrieved, they // sometimes are indicative the file has grown before the page is diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index 68ad34bf..f704814f 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -37,7 +37,7 @@ public class RateLimitGuard extends Guard { private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); - long lastFailureTime = 0; + protected long lastFailureTime = 0; public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) throws IllegalArgumentException { super(context, scheme, realm); diff --git a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java index 101b2b30..10947e7b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java @@ -37,7 +37,7 @@ import org.restlet.resource.Variant; * @contributor gojomo */ public class ReportGenResource extends JobRelatedResource { - String reportClass; + protected String reportClass; public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException { super(ctx, req, res); diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 32ee03e6..ab8247c6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -62,9 +62,9 @@ import org.restlet.resource.WriterRepresentation; * @contributor gojomo */ public class ScriptResource extends JobRelatedResource { - static ScriptEngineManager MANAGER = new ScriptEngineManager(); + protected static ScriptEngineManager MANAGER = new ScriptEngineManager(); // oddly, ordering is different each call to getEngineFactories, so cache - static LinkedList FACTORIES = new LinkedList(); + protected static LinkedList FACTORIES = new LinkedList(); static { FACTORIES.addAll(MANAGER.getEngineFactories()); // Sort factories alphabetically so that they appear in the UI consistently @@ -75,12 +75,12 @@ public class ScriptResource extends JobRelatedResource { } }); } - String script = ""; - Exception ex = null; - int linesExecuted = 0; - String rawOutput = ""; - String htmlOutput = ""; - String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0); + protected String script = ""; + protected Exception ex = null; + protected int linesExecuted = 0; + protected String rawOutput = ""; + protected String htmlOutput = ""; + protected String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0); public ScriptResource(Context ctx, Request req, Response res) throws ResourceException { super(ctx, req, res); diff --git a/engine/src/main/java/org/archive/crawler/spring/DecideRuledSheetAssociation.java b/engine/src/main/java/org/archive/crawler/spring/DecideRuledSheetAssociation.java index 401d38de..b6e758d2 100644 --- a/engine/src/main/java/org/archive/crawler/spring/DecideRuledSheetAssociation.java +++ b/engine/src/main/java/org/archive/crawler/spring/DecideRuledSheetAssociation.java @@ -32,8 +32,8 @@ import org.springframework.core.Ordered; */ public class DecideRuledSheetAssociation extends SheetAssociation implements Ordered, Comparable, BeanNameAware { - DecideRule rules; - int order = 0; + protected DecideRule rules; + protected int order = 0; public DecideRule getRules() { return rules; @@ -59,7 +59,7 @@ implements Ordered, Comparable, BeanNameAware { return name.compareTo(o.name); } - String name; + protected String name; public void setBeanName(String name) { this.name = name; } diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetAssociation.java b/engine/src/main/java/org/archive/crawler/spring/SheetAssociation.java index 675354f0..1a3d57dd 100644 --- a/engine/src/main/java/org/archive/crawler/spring/SheetAssociation.java +++ b/engine/src/main/java/org/archive/crawler/spring/SheetAssociation.java @@ -32,7 +32,7 @@ import java.util.List; * @contributor gojomo */ public abstract class SheetAssociation { - List targetSheetNames = new LinkedList(); + protected List targetSheetNames = new LinkedList(); public List getTargetSheetNames() { return targetSheetNames; } diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java index fee9e81c..80775e19 100644 --- a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java +++ b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java @@ -57,14 +57,14 @@ BeanFactoryAware, OverlayMapsSource, ApplicationListener { private static final Logger logger = Logger.getLogger(SheetOverlaysManager.class.getName()); - BeanFactory beanFactory; + protected BeanFactory beanFactory; /** all SheetAssociations by DecideRule evaluation */ - SortedSet ruleAssociations = + protected SortedSet ruleAssociations = new ConcurrentSkipListSet(); - NavigableMap> sheetNamesBySurt = new ConcurrentSkipListMap>(); + protected NavigableMap> sheetNamesBySurt = new ConcurrentSkipListMap>(); /** all sheets by (bean)name*/ - Map sheetsByName = new ConcurrentHashMap(); + protected Map sheetsByName = new ConcurrentHashMap(); public void setBeanFactory(BeanFactory beanFactory) throws BeansException { this.beanFactory = beanFactory; diff --git a/engine/src/main/java/org/archive/crawler/spring/SurtPrefixesSheetAssociation.java b/engine/src/main/java/org/archive/crawler/spring/SurtPrefixesSheetAssociation.java index c9d6802a..14b47b36 100644 --- a/engine/src/main/java/org/archive/crawler/spring/SurtPrefixesSheetAssociation.java +++ b/engine/src/main/java/org/archive/crawler/spring/SurtPrefixesSheetAssociation.java @@ -29,7 +29,7 @@ import org.springframework.beans.factory.annotation.Required; * @contributor gojomo */ public class SurtPrefixesSheetAssociation extends SheetAssociation { - List surtPrefixes; + protected List surtPrefixes; public List getSurtPrefixes() { return surtPrefixes; diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index af16ffae..11ed6677 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -91,7 +91,7 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean { this.bdb = bdb; } - String beanName; + protected String beanName; public void setBeanName(String name) { this.beanName = name; } @@ -99,7 +99,7 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean { public BdbUriUniqFilter() { } - boolean isRunning = false; + protected boolean isRunning = false; public void start() { if(isRunning()) { return; @@ -391,7 +391,7 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean { } } public void finishCheckpoint(Checkpoint checkpointInProgress) {} - Checkpoint recoveryCheckpoint; + protected Checkpoint recoveryCheckpoint; public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { this.recoveryCheckpoint = recoveryCheckpoint; } diff --git a/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java b/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java index 5be5b9e8..90ae8f26 100644 --- a/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java +++ b/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java @@ -39,7 +39,7 @@ public class BenchmarkUriUniqFilters implements UriUniqFilter.CrawlUriReceiver { // Logger.getLogger(BenchmarkUriUniqFilters.class.getName()); private BufferedWriter out; // optional to dump uniq items - String current; // current line/URI being checked + protected String current; // current line/URI being checked /** * Test the UriUniqFilter implementation (MemUriUniqFilter, diff --git a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java index 98120a6e..4ab528ac 100644 --- a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java @@ -60,7 +60,7 @@ implements Serializable, InitializingBean { private static Logger LOGGER = Logger.getLogger(BloomUriUniqFilter.class.getName()); - BloomFilter bloom; // package access for testing convenience + protected BloomFilter bloom; // package access for testing convenience public BloomFilter getBloomFilter() { return bloom; } diff --git a/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java index a44f3ac6..9bb54a43 100644 --- a/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java @@ -41,13 +41,13 @@ import org.archive.util.ArchiveUtils; * @author gojomo */ public class DiskFPMergeUriUniqFilter extends FPMergeUriUniqFilter { - long count = 0; - File scratchDir; - File currentFps; - File newFpsFile; - DataOutputStream newFps; - long newCount; - DataInputStream oldFps; + protected long count = 0; + protected File scratchDir; + protected File currentFps; + protected File newFpsFile; + protected DataOutputStream newFps; + protected long newCount; + protected DataInputStream oldFps; public DiskFPMergeUriUniqFilter(File scratchDir) { super(); diff --git a/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java index 8ca2028c..c1d7183c 100644 --- a/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java @@ -39,7 +39,7 @@ implements Serializable { private transient FPGenerator fpgen = FPGenerator.std64; - LongFPSet fpset; + protected LongFPSet fpset; public LongFPSet getFpset() { return this.fpset; } diff --git a/engine/src/main/java/org/archive/crawler/util/Logs.java b/engine/src/main/java/org/archive/crawler/util/Logs.java index f7c00df9..3aa08b3e 100644 --- a/engine/src/main/java/org/archive/crawler/util/Logs.java +++ b/engine/src/main/java/org/archive/crawler/util/Logs.java @@ -33,7 +33,7 @@ public enum Logs{ NONFATAL_ERRORS ("nonfatal-errors.log"), URI_ERRORS ("uri-errors.log"); - String filename; + protected String filename; Logs(String filename){ this.filename = filename; diff --git a/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java index d9c59276..55f0295f 100644 --- a/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java @@ -30,7 +30,7 @@ import java.util.HashSet; public class MemUriUniqFilter extends SetBasedUriUniqFilter { private static final long serialVersionUID = 1L; - HashSet hashSet; + protected HashSet hashSet; protected synchronized boolean setAdd(CharSequence uri) { return hashSet.add(uri); diff --git a/engine/src/main/java/org/archive/crawler/util/TopNSet.java b/engine/src/main/java/org/archive/crawler/util/TopNSet.java index 65fdfbf4..4a8b2599 100644 --- a/engine/src/main/java/org/archive/crawler/util/TopNSet.java +++ b/engine/src/main/java/org/archive/crawler/util/TopNSet.java @@ -47,12 +47,12 @@ public class TopNSet implements Serializable { private static final long serialVersionUID = 1L; - int maxsize; - ConcurrentMap set; - volatile long smallestKnownValue; - volatile String smallestKnownKey; - volatile long largestKnownValue; - volatile String largestKnownKey; + protected int maxsize; + protected ConcurrentMap set; + protected volatile long smallestKnownValue; + protected volatile String smallestKnownKey; + protected volatile long largestKnownValue; + protected volatile String largestKnownKey; public TopNSet(int size){ maxsize = size; diff --git a/modules/src/main/java/org/archive/modules/CrawlMetadata.java b/modules/src/main/java/org/archive/modules/CrawlMetadata.java index 5b174a59..6390ff27 100644 --- a/modules/src/main/java/org/archive/modules/CrawlMetadata.java +++ b/modules/src/main/java/org/archive/modules/CrawlMetadata.java @@ -49,7 +49,7 @@ implements UserAgentProvider, InitializingBean { private static final long serialVersionUID = 1L; - KeyedProperties kp = new KeyedProperties(); + protected KeyedProperties kp = new KeyedProperties(); public KeyedProperties getKeyedProperties() { return kp; } @@ -71,7 +71,7 @@ implements UserAgentProvider, /** Map of all available RobotsPolicies, by name, to choose from. * assembled from declared instances in configuration plus the standard * 'obey' (aka 'classic') and 'ignore' policies. */ - Map availableRobotsPolicies = new HashMap(); + protected Map availableRobotsPolicies = new HashMap(); public Map getAvailableRobotsPolicies() { return availableRobotsPolicies; } @@ -95,7 +95,7 @@ implements UserAgentProvider, return availableRobotsPolicies.get(getRobotsPolicyName()); } - String operator = ""; + protected String operator = ""; public String getOperator() { return operator; } @@ -103,7 +103,7 @@ implements UserAgentProvider, this.operator = operatorName; } - String description = ""; + protected String description = ""; public String getDescription() { return description; } @@ -155,7 +155,7 @@ implements UserAgentProvider, } - String audience = ""; + protected String audience = ""; public String getAudience() { return audience; } @@ -163,7 +163,7 @@ implements UserAgentProvider, this.audience = audience; } - String organization = ""; + protected String organization = ""; public String getOrganization() { return organization; } @@ -180,7 +180,7 @@ implements UserAgentProvider, return userAgent; } - String jobName; + protected String jobName; public String getJobName() { return jobName; } @@ -198,7 +198,7 @@ implements UserAgentProvider, ensureStandardPoliciesAvailable(); } - static Validator VALIDATOR = new BeanFieldsPatternValidator( + protected static Validator VALIDATOR = new BeanFieldsPatternValidator( CrawlMetadata.class, "userAgentTemplate", "^.*\\+@OPERATOR_CONTACT_URL@.*$", diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index e1846bb6..26d4f5b0 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -176,10 +176,6 @@ implements MultiReporter, Serializable, OverlayContext { // User agent to masquerade as when crawling this URI. If null, globals should be used private String userAgent = null; - - // From header - // TODO: This and user-agent really belong in FetchHTTP -// private transient String from = null; // Once a link extractor has finished processing this curi this will be // set as true @@ -187,7 +183,6 @@ implements MultiReporter, Serializable, OverlayContext { transient private int discardedOutlinks = 0; -//////////////////////////////////////////////////////////////////// private long contentSize = UNCALCULATED; private long contentLength = UNCALCULATED; @@ -201,17 +196,8 @@ implements MultiReporter, Serializable, OverlayContext { * {@link CoreAttributeConstants} interface. Use this list to carry * data or state produced by custom processors rather change the * classes {@link CrawlURI} or this class, CrawlURI. - * - * Transient to allow more efficient custom serialization. - * - * Package-protected so CrawlURI can access it directly. */ - Map data; - - -// private transient SheetManager manager; -// private transient StateProvider provider; - + protected Map data; private boolean forceRevisit = false; // even if already visited @@ -1012,8 +998,8 @@ implements MultiReporter, Serializable, OverlayContext { return Base32.encode(this.contentDigest); } - transient Object holder; - transient Object holderKey; + transient protected Object holder; + transient protected Object holderKey; /** * Remember a 'holder' to which some enclosing/queueing @@ -1070,7 +1056,7 @@ implements MultiReporter, Serializable, OverlayContext { /** spot for an integer cost to be placed by external facility (frontier). * cost is truncated to 8 bits at times, so should not exceed 255 */ - int holderCost = UNCALCULATED; + protected int holderCost = UNCALCULATED; /** * Return the 'holderCost' for convenience of external facility (frontier) * @return value of holderCost @@ -1094,10 +1080,9 @@ implements MultiReporter, Serializable, OverlayContext { * The LinksScoper processor converts Link instances in this collection * to CrawlURI instances. */ - transient Collection outLinks = new HashSet(); + protected transient Collection outLinks = new HashSet(); - - transient Collection outCandidates = new HashSet(); + protected transient Collection outCandidates = new HashSet(); /** * Returns discovered links. The returned collection might be empty if @@ -1789,7 +1774,7 @@ implements MultiReporter, Serializable, OverlayContext { return this.politenessDelay; } - transient CrawlURI fullVia; + transient protected CrawlURI fullVia; public void setFullVia(CrawlURI curi) { this.fullVia = curi; } diff --git a/modules/src/main/java/org/archive/modules/Processor.java b/modules/src/main/java/org/archive/modules/Processor.java index 8f720293..38bf2d46 100644 --- a/modules/src/main/java/org/archive/modules/Processor.java +++ b/modules/src/main/java/org/archive/modules/Processor.java @@ -57,7 +57,7 @@ implements HasKeyedProperties, return kp; } - String beanName; + protected String beanName; public String getBeanName() { return this.beanName; } @@ -249,7 +249,7 @@ implements HasKeyedProperties, return "Processor: "+getClass().getName()+"\n"; } - boolean isRunning = false; + protected boolean isRunning = false; public boolean isRunning() { return isRunning; } diff --git a/modules/src/main/java/org/archive/modules/ProcessorChain.java b/modules/src/main/java/org/archive/modules/ProcessorChain.java index 7a3029b9..6e8bb068 100644 --- a/modules/src/main/java/org/archive/modules/ProcessorChain.java +++ b/modules/src/main/java/org/archive/modules/ProcessorChain.java @@ -30,7 +30,7 @@ implements Iterable, Reporter, Lifecycle { - KeyedProperties kp = new KeyedProperties(); + protected KeyedProperties kp = new KeyedProperties(); public KeyedProperties getKeyedProperties() { return kp; } @@ -51,7 +51,7 @@ implements Iterable, kp.put("processors",processors); } - boolean isRunning = false; + protected boolean isRunning = false; public boolean isRunning() { return isRunning; } diff --git a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java index 802de482..ac983dfe 100644 --- a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java +++ b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java @@ -73,7 +73,7 @@ implements ApplicationContextAware, InitializingBean { this.engineName = name; } - ReadSource scriptSource = null; + protected ReadSource scriptSource = null; public ReadSource getScriptSource() { return this.scriptSource; } @@ -96,7 +96,7 @@ implements ApplicationContextAware, InitializingBean { this.isolateThreads = isolateThreads; } - ApplicationContext appCtx; + protected ApplicationContext appCtx; public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.appCtx = applicationContext; } diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index 9f71e85d..03b7895c 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -53,7 +53,7 @@ public abstract class Credential implements Serializable { /** * The root domain this credential goes against: E.g. www.archive.org */ - String domain = ""; + protected String domain = ""; /** * @param context Context to use when searching for credential domain. * @return The domain/root URI this credential is to go against. diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java index fdbf99b0..83c9deea 100644 --- a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java +++ b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java @@ -58,7 +58,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties { private static Logger logger = Logger.getLogger( "org.archive.crawler.datamodel.CredentialStore"); - KeyedProperties kp = new KeyedProperties(); + protected KeyedProperties kp = new KeyedProperties(); public KeyedProperties getKeyedProperties() { return kp; } diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index 563ad1a4..258f1820 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -52,7 +52,7 @@ public class HtmlFormCredential extends Credential { * Full URI of page that contains the HTML login form we're to apply these * credentials too: E.g. http://www.archive.org */ - String loginUri = ""; + protected String loginUri = ""; public String getLoginUri() { return this.loginUri; } @@ -63,7 +63,7 @@ public class HtmlFormCredential extends Credential { /** * Form items. */ - Map formItems = new HashMap(); + protected Map formItems = new HashMap(); public Map getFormItems() { return this.formItems; } @@ -79,7 +79,7 @@ public class HtmlFormCredential extends Credential { /** * GET or POST. */ - Method httpMethod = Method.POST; + protected Method httpMethod = Method.POST; public Method getHttpMethod() { return this.httpMethod; } diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index 6491f199..7035e1c9 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -48,7 +48,7 @@ public class HttpAuthenticationCredential extends Credential { /** Basic/Digest Auth realm. */ - String realm = "Realm"; + protected String realm = "Realm"; public String getRealm() { return this.realm; } @@ -57,7 +57,7 @@ public class HttpAuthenticationCredential extends Credential { } /** Login. */ - String login = "login"; + protected String login = "login"; public String getLogin() { return this.login; } @@ -66,7 +66,7 @@ public class HttpAuthenticationCredential extends Credential { } /** Password. */ - String password = "password"; + protected String password = "password"; public String getPassword() { return this.password; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java index 3d1fc0a3..be454e05 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java @@ -55,7 +55,7 @@ public class ExternalGeoLocationDecideRule extends PredicatedDecideRule { private static final Logger LOGGER = Logger.getLogger(ExternalGeoLocationDecideRule.class.getName()); - ExternalGeoLookupInterface lookup = null; + protected ExternalGeoLookupInterface lookup = null; public ExternalGeoLookupInterface getLookup() { return this.lookup; } @@ -66,7 +66,7 @@ public class ExternalGeoLocationDecideRule extends PredicatedDecideRule { /** * Country code name. */ - List countryCodes = new ArrayList(); + protected List countryCodes = new ArrayList(); public List getCountryCodes() { return this.countryCodes; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java index b0f67fe4..818d8fa9 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java @@ -33,7 +33,7 @@ public class FetchStatusDecideRule extends PredicatedDecideRule { private static final long serialVersionUID = 3L; - List statusCodes = new ArrayList(); + protected List statusCodes = new ArrayList(); public List getStatusCodes() { return this.statusCodes; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ResourceNoLongerThanDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ResourceNoLongerThanDecideRule.java index b977710c..3571ca78 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ResourceNoLongerThanDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ResourceNoLongerThanDecideRule.java @@ -123,7 +123,7 @@ public class ResourceNoLongerThanDecideRule extends PredicatedDecideRule { } - boolean test(int contentlength) { + protected boolean test(int contentlength) { return contentlength < getContentLengthThreshold(); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java index 0ab13efe..3092bbe6 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java @@ -96,7 +96,7 @@ implements ApplicationContextAware, InitializingBean { this.isolateThreads = isolateThreads; } - ApplicationContext appCtx; + protected ApplicationContext appCtx; public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.appCtx = applicationContext; } diff --git a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java index 91cbcfc4..348e1c5b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java @@ -34,7 +34,7 @@ public class AggressiveExtractorHTML extends ExtractorHTML { private static final long serialVersionUID = 3L; - static Logger logger = + protected static Logger logger = Logger.getLogger(AggressiveExtractorHTML.class.getName()); public AggressiveExtractorHTML() { diff --git a/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java b/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java index 6706bd68..b0f61ada 100644 --- a/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java +++ b/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java @@ -36,7 +36,7 @@ import com.anotherbigidea.flash.writers.SWFTagTypesImpl; */ @SuppressWarnings("unchecked") public class CustomSWFTags extends SWFTagTypesImpl { - SWFActions actions; + protected SWFActions actions; public CustomSWFTags(SWFActions a) { super(null); diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java index 2d8e456d..44cfd9c6 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -75,7 +75,7 @@ public abstract class Extractor extends Processor { { setExtractorParameters(DEFAULT_PARAMETERS); } - transient ExtractorParameters extractorParameters; + protected transient ExtractorParameters extractorParameters; public ExtractorParameters getExtractorParameters() { return extractorParameters; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index d10a6c06..85dfc543 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -66,7 +66,7 @@ public class ExtractorCSS extends ContentExtractor { // CSS escapes: "Parentheses, commas, whitespace characters, single // quotes (') and double quotes (") appearing in a URL must be // escaped with a backslash" - static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])"; + protected static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])"; /** * CSS URL extractor pattern. @@ -75,7 +75,7 @@ public class ExtractorCSS extends ContentExtractor { **/ // static final String CSS_URI_EXTRACTOR = // "url[(]\\s*([\"\']?)([^\\\"\\'].*?)\\1\\s*[)]"; - static final String CSS_URI_EXTRACTOR = + protected static final String CSS_URI_EXTRACTOR = "(?i)(?:@import (?:url[(]|)|url[(])\\s*([\\\"\']?)" + // G1 "([^\\\"\'].{0,"+UURI.MAX_URL_LENGTH+"}?)\\1\\s*[);]"; // G2 // GROUPS: diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index e5d4ada3..5bbe0675 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -777,7 +777,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean * is TRANSIENT data. Make a copy if you want the data to live outside * of this extractors' lifetime. */ - void extract(CrawlURI curi, CharSequence cs) { + protected void extract(CrawlURI curi, CharSequence cs) { Matcher tags = TextUtils.getMatcher(relevantTagPattern,cs); while(tags.find()) { if(Thread.interrupted()){ diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 2b321543..714ff763 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -40,7 +40,7 @@ public class ExtractorHTTP extends Extractor { } /** should all HTTP URIs be used to infer a link to the site's root? */ - boolean inferRootPage = false; + protected boolean inferRootPage = false; public boolean getInferRootPage() { return inferRootPage; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 8c65e465..994b0a1d 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -65,7 +65,7 @@ public class ExtractorJS extends ContentExtractor { // finds whitespace-free strings in Javascript // (areas between paired ' or " characters, possibly backslash-quoted // on the ends, but not in the middle) - static final String JAVASCRIPT_STRING_EXTRACTOR = + protected static final String JAVASCRIPT_STRING_EXTRACTOR = "(\\\\{0,8}+(?:\"|\'))(\\S{0,"+UURI.MAX_URL_LENGTH+"}?)(?:\\1)"; // GROUPS: // (G1) ' or " with optional leading backslashes diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java index 4d881a21..fd2d31ce 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java @@ -53,7 +53,7 @@ public class ExtractorSWF extends ContentExtractor { private static final int MAX_READ_SIZE = 1024 * 1024; // 1MB - static final String JSSTRING = "javascript:"; + protected static final String JSSTRING = "javascript:"; /** * @param name @@ -288,7 +288,7 @@ public class ExtractorSWF extends ContentExtractor { */ public class CrawlUriSWFAction extends SWFActionsImpl { - CrawlURI curi; + protected CrawlURI curi; private long linkCount; private Extractor ext; diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java index 593cb17c..3b3a0529 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java @@ -53,7 +53,7 @@ public class ExtractorURI extends Extractor { private static Logger LOGGER = Logger.getLogger(ExtractorURI.class.getName()); - static final String ABS_HTTP_URI_PATTERN = "^https?://[^\\s<>]*$"; + protected static final String ABS_HTTP_URI_PATTERN = "^https?://[^\\s<>]*$"; /** * Constructor diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java index 14b64c27..110f2cc1 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java @@ -70,7 +70,7 @@ public class ExtractorUniversal extends ContentExtractor { * 3 chars seperated by 3 dots). Does not ensure that the numbers are * each in the range 0-255. */ - static final Pattern IP_ADDRESS = Pattern.compile( + protected static final Pattern IP_ADDRESS = Pattern.compile( "((http://)|(https://))(\\d(\\d)?(\\d)?\\.\\d(\\d)?(\\d)?\\.\\d(\\d)?(\\d)?\\.\\d(\\d)?(\\d)?)"); /** diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index 8e1069a4..3fcbed63 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -428,7 +428,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { * @param cs * Sequence from underlying ReplayCharSequence. */ - void extract(CrawlURI curi, CharSequence cs) { + protected void extract(CrawlURI curi, CharSequence cs) { Source source = new Source(cs); List elements = source.findAllElements(StartTagType.NORMAL); for (Iterator elementIter = elements.iterator(); diff --git a/modules/src/main/java/org/archive/modules/extractor/PDFParser.java b/modules/src/main/java/org/archive/modules/extractor/PDFParser.java index 20f9d78e..9a44d59b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/PDFParser.java +++ b/modules/src/main/java/org/archive/modules/extractor/PDFParser.java @@ -41,18 +41,18 @@ import java.util.*; // to parse small, but admittedly complex, documents. public class PDFParser { - ArrayList foundURIs; - ArrayList> encounteredReferences; - PdfReader documentReader; - byte[] document; - PdfDictionary catalog; + protected ArrayList foundURIs; + protected ArrayList> encounteredReferences; + protected PdfReader documentReader; + protected byte[] document; + protected PdfDictionary catalog; public PDFParser(String doc) throws IOException { resetState(); getInFromFile(doc); initialize(); } - public PDFParser(byte[] doc) throws IOException{ + public PDFParser(byte[] doc) throws IOException{ resetState(); document = doc; initialize(); diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java index 47a095e8..ace25cb3 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java @@ -70,7 +70,7 @@ public abstract class AbstractCookieStorage this.cookiesSaveFile = cookiesSaveFile; } - boolean isRunning = false; + protected boolean isRunning = false; public void start() { if(isRunning()) { return; diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java index f1aedae2..de91f5f8 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java @@ -52,7 +52,7 @@ public class BdbCookieStorage extends AbstractCookieStorage implements Checkpoin } /** are we a checkpoint recovery? (in which case, reuse stored cookie data?) */ - boolean isCheckpointRecovery = false; + protected boolean isCheckpointRecovery = false; public static String COOKIEDB_NAME = "http_cookies"; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 40340fbd..7acd541a 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -74,7 +74,7 @@ public class FetchDNS extends Processor { * If a DNS lookup fails, whether or not to fallback to InetAddress * resolution, which may use local 'hosts' files or other mechanisms. */ - boolean acceptNonDnsResolves = false; + protected boolean acceptNonDnsResolves = false; public boolean getAcceptNonDnsResolves() { return acceptNonDnsResolves; } @@ -112,7 +112,7 @@ public class FetchDNS extends Processor { * Which algorithm (for example MD5 or SHA-1) to use to perform an * on-the-fly digest hash of retrieved content-bodies. */ - String digestAlgorithm = "sha1"; + protected String digestAlgorithm = "sha1"; public String getDigestAlgorithm() { return digestAlgorithm; } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java index ec0f22d2..77275cd6 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java @@ -159,7 +159,7 @@ public class FetchFTP extends Processor { * Which algorithm (for example MD5 or SHA-1) to use to perform an * on-the-fly digest hash of retrieved content-bodies. */ - String digestAlgorithm = "sha1"; + protected String digestAlgorithm = "sha1"; public String getDigestAlgorithm() { return digestAlgorithm; } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 13238a13..c339b0f4 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -275,7 +275,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * Which algorithm (for example MD5 or SHA-1) to use to perform an * on-the-fly digest hash of retrieved content-bodies. */ - String digestAlgorithm = "sha1"; + protected String digestAlgorithm = "sha1"; public String getDigestAlgorithm() { return digestAlgorithm; } @@ -479,7 +479,7 @@ public class FetchHTTP extends Processor implements Lifecycle { public static final String HTTPS_SCHEME = "https"; - CookieStorage cookieStorage = new BdbCookieStorage(); + protected CookieStorage cookieStorage = new BdbCookieStorage(); @Autowired(required=false) public void setCookieStorage(CookieStorage storage) { this.cookieStorage = storage; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java index 861caf25..f1c98511 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java @@ -49,27 +49,27 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter public void tally(CrawlURI curi, Stage stage); } - long totalScheduled; // anything initially scheduled + protected long totalScheduled; // anything initially scheduled // (totalScheduled - (fetchSuccesses + fetchFailures) - long fetchSuccesses; // anything disposed-success + protected long fetchSuccesses; // anything disposed-success // (HTTP 2XX response codes, other non-errors) - long fetchFailures; // anything disposed-failure - long fetchDisregards; // anything disposed-disregard - long fetchResponses; // all positive responses (incl. 3XX, 4XX, 5XX) - long robotsDenials; // all robots-precluded failures - long successBytes; // total size of all success responses - long totalBytes; // total size of all responses - long fetchNonResponses; // processing attempts resulting in no response + protected long fetchFailures; // anything disposed-failure + protected long fetchDisregards; // anything disposed-disregard + protected long fetchResponses; // all positive responses (incl. 3XX, 4XX, 5XX) + protected long robotsDenials; // all robots-precluded failures + protected long successBytes; // total size of all success responses + protected long totalBytes; // total size of all responses + protected long fetchNonResponses; // processing attempts resulting in no response // (both failures and temp deferrals) - long novelBytes; - long novelUrls; - long notModifiedBytes; - long notModifiedUrls; - long dupByHashBytes; - long dupByHashUrls; + protected long novelBytes; + protected long novelUrls; + protected long notModifiedBytes; + protected long notModifiedUrls; + protected long dupByHashBytes; + protected long dupByHashUrls; - long lastSuccessTime; + protected long lastSuccessTime; public synchronized void tally(CrawlURI curi, Stage stage) { switch(stage) { diff --git a/modules/src/main/java/org/archive/modules/net/BdbServerCache.java b/modules/src/main/java/org/archive/modules/net/BdbServerCache.java index 2c5d8320..3ca49191 100644 --- a/modules/src/main/java/org/archive/modules/net/BdbServerCache.java +++ b/modules/src/main/java/org/archive/modules/net/BdbServerCache.java @@ -58,7 +58,7 @@ implements Lifecycle { isRunning = true; } - boolean isRunning = false; + protected boolean isRunning = false; public boolean isRunning() { return isRunning; } diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index e314c965..f2330b1d 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -67,9 +67,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden private String server; // actually, host+port in the https case private int port; protected Robotstxt robotstxt; - long robotsFetched = ROBOTS_NOT_FETCHED; - boolean validRobots = false; - FetchStats substats = new FetchStats(); + protected long robotsFetched = ROBOTS_NOT_FETCHED; + protected boolean validRobots = false; + protected FetchStats substats = new FetchStats(); // how many consecutive connection errors have been encountered; // used to drive exponentially increasing retry timeout or decision diff --git a/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java index 988dd3bc..e3ac7daa 100644 --- a/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java @@ -36,10 +36,10 @@ import org.archive.spring.ConfigString; */ public class CustomRobotsPolicy extends RobotsPolicy { - Robotstxt customRobotstxt; + protected Robotstxt customRobotstxt; /** textual alternate robots.txt rules to follow */ - ReadSource customRobots = new ConfigString(""); + protected ReadSource customRobots = new ConfigString(""); public ReadSource getCustomRobots() { return customRobots; } @@ -49,7 +49,7 @@ public class CustomRobotsPolicy extends RobotsPolicy { } /** whether to obey the 'nofollow' directive in an HTML META ROBOTS element */ - boolean obeyMetaRobotsNofollow = true; + protected boolean obeyMetaRobotsNofollow = true; public boolean isObeyMetaRobotsNofollow() { return obeyMetaRobotsNofollow; } diff --git a/modules/src/main/java/org/archive/modules/net/DefaultTempDirProvider.java b/modules/src/main/java/org/archive/modules/net/DefaultTempDirProvider.java index 90d4bdda..dfaa2bee 100644 --- a/modules/src/main/java/org/archive/modules/net/DefaultTempDirProvider.java +++ b/modules/src/main/java/org/archive/modules/net/DefaultTempDirProvider.java @@ -30,14 +30,11 @@ import org.archive.modules.extractor.TempDirProvider; */ public class DefaultTempDirProvider implements TempDirProvider { - private static final long serialVersionUID = 1L; - final private static File TEMP_DIR = makeTempDir(); - - static File makeTempDir() { + protected static File makeTempDir() { File f; try { f = File.createTempFile("xxx", null); diff --git a/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java index 4d28ec4a..b2c05321 100644 --- a/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java @@ -48,7 +48,7 @@ import org.archive.modules.CrawlURI; public class FirstNamedRobotsPolicy extends RobotsPolicy { /** list of user-agents to try; if any are allowed, a URI will be crawled */ - List candidateUserAgents = new LinkedList(); + protected List candidateUserAgents = new LinkedList(); public List getCandidateUserAgents() { return candidateUserAgents; } @@ -57,7 +57,7 @@ public class FirstNamedRobotsPolicy extends RobotsPolicy { } /** whether to adopt the user-agent that is allowed for the fetch */ - boolean shouldMasquerade = true; + protected boolean shouldMasquerade = true; public boolean getShouldMasquerade() { return shouldMasquerade; } @@ -66,7 +66,7 @@ public class FirstNamedRobotsPolicy extends RobotsPolicy { } /** whether to obey the 'nofollow' directive in an HTML META ROBOTS element */ - boolean obeyMetaRobotsNofollow = true; + protected boolean obeyMetaRobotsNofollow = true; public boolean isObeyMetaRobotsNofollow() { return obeyMetaRobotsNofollow; } diff --git a/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java index 2d54d997..5c60c096 100644 --- a/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java @@ -44,7 +44,7 @@ import org.archive.modules.CrawlURI; public class MostFavoredRobotsPolicy extends RobotsPolicy { /** list of user-agents to try; if any are allowed, a URI will be crawled */ - List candidateUserAgents = new LinkedList(); + protected List candidateUserAgents = new LinkedList(); public List getCandidateUserAgents() { return candidateUserAgents; } @@ -53,7 +53,7 @@ public class MostFavoredRobotsPolicy extends RobotsPolicy { } /** whether to adopt the user-agent that is allowed for the fetch */ - boolean shouldMasquerade = true; + protected boolean shouldMasquerade = true; public boolean getShouldMasquerade() { return shouldMasquerade; } @@ -62,7 +62,7 @@ public class MostFavoredRobotsPolicy extends RobotsPolicy { } /** whether to obey the 'nofollow' directive in an HTML META ROBOTS element */ - boolean obeyMetaRobotsNofollow = true; + protected boolean obeyMetaRobotsNofollow = true; public boolean isObeyMetaRobotsNofollow() { return obeyMetaRobotsNofollow; } diff --git a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java index 5ded5446..97752a0e 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java @@ -32,9 +32,9 @@ import com.esotericsoftware.kryo.serialize.ReferenceFieldSerializer; public class RobotsDirectives implements Serializable { private static final long serialVersionUID = 5386542759286155383L; - ConcurrentSkipListSet disallows = new ConcurrentSkipListSet(); - ConcurrentSkipListSet allows = new ConcurrentSkipListSet(); - float crawlDelay = -1; + protected ConcurrentSkipListSet disallows = new ConcurrentSkipListSet(); + protected ConcurrentSkipListSet allows = new ConcurrentSkipListSet(); + protected float crawlDelay = -1; public boolean allows(String path) { return !(longestPrefixLength(disallows, path) > longestPrefixLength(allows, path)); diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index 32ac0e5f..6ec60571 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -47,15 +47,15 @@ public class Robotstxt implements Serializable { // all user agents contained in this robots.txt // in order of declaration // TODO: consider discarding irrelevant entries - LinkedList namedUserAgents = new LinkedList(); + protected LinkedList namedUserAgents = new LinkedList(); // map user-agents to directives - Map agentsToDirectives = + protected Map agentsToDirectives = new HashMap(); - RobotsDirectives wildcardDirectives = null; + protected RobotsDirectives wildcardDirectives = null; - boolean hasErrors = false; + protected boolean hasErrors = false; - static RobotsDirectives NO_DIRECTIVES = new RobotsDirectives(); + protected static RobotsDirectives NO_DIRECTIVES = new RobotsDirectives(); /** empty, reusable instance for all sites providing no rules */ public static Robotstxt NO_ROBOTS = new Robotstxt(); diff --git a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java index 252d7e76..c3282b0c 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java @@ -39,7 +39,7 @@ public class FetchHistoryProcessor extends Processor { private static final long serialVersionUID = 1L; /** Desired history array length. */ - int historyLength = 2; + protected int historyLength = 2; public int getHistoryLength() { return this.historyLength; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java index de3e2320..b9220fcd 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java @@ -54,7 +54,7 @@ public class PersistLoadProcessor extends PersistOnlineProcessor { * the followup crawl(s).) Only one of {@preloadSource} and * {@preloadSourceUrl} may be specified. */ - ConfigPath preloadSource = + protected ConfigPath preloadSource = new ConfigFile("preload source",""); public ConfigPath getPreloadSource() { return preloadSource; @@ -72,7 +72,7 @@ public class PersistLoadProcessor extends PersistOnlineProcessor { * and then copy and reuse that environment in the followup crawl(s).) * Only one of {@preloadSource} and {@preloadSourceUrl} may be specified. */ - String preloadSourceUrl = ""; + protected String preloadSourceUrl = ""; public String getPreloadSourceUrl() { return preloadSourceUrl; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java index 7e397b80..3e8a3a6f 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java @@ -49,7 +49,7 @@ implements Checkpointable, Lifecycle { // description: "Filename to which to log URI persistence information. " + // "Default is 'logs/persistlog.txtser.gz'. " - ConfigPath logFile = new ConfigPath("URI persistence log file","${launchId}/logs/persistlog.txtser.gz"); + protected ConfigPath logFile = new ConfigPath("URI persistence log file","${launchId}/logs/persistlog.txtser.gz"); public ConfigPath getLogFile() { return this.logFile; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java index ec40ead7..6ced89a4 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java @@ -48,7 +48,7 @@ implements Lifecycle { this.bdb = bdb; } - String historyDbName = "uri_history"; + protected String historyDbName = "uri_history"; public String getHistoryDbName() { return this.historyDbName; } diff --git a/modules/src/main/java/org/archive/modules/seeds/SeedModule.java b/modules/src/main/java/org/archive/modules/seeds/SeedModule.java index 93864ef5..4b76b8d5 100644 --- a/modules/src/main/java/org/archive/modules/seeds/SeedModule.java +++ b/modules/src/main/java/org/archive/modules/seeds/SeedModule.java @@ -37,7 +37,7 @@ public abstract class SeedModule implements Serializable * from that seed. When present, such source tags appear in the * second-to-last crawl.log field. */ - boolean sourceTagSeeds = false; + protected boolean sourceTagSeeds = false; public boolean getSourceTagSeeds() { return sourceTagSeeds; } diff --git a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java index eee32184..efedbbaa 100644 --- a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java @@ -94,7 +94,7 @@ public class Kw3WriterProcessor extends Processor { /** * Top-level directory for archive files. */ - ConfigPath path = new ConfigPath("Kw3Writer subdirectory","arcs"); + protected ConfigPath path = new ConfigPath("Kw3Writer subdirectory","arcs"); public ConfigPath getPath() { return this.path; } @@ -105,7 +105,7 @@ public class Kw3WriterProcessor extends Processor { /** * Max size for each file. */ - long maxFileSizeBytes = 100000000L; + protected long maxFileSizeBytes = 100000000L; public long getMaxFileSizeBytes() { return maxFileSizeBytes; } @@ -116,7 +116,7 @@ public class Kw3WriterProcessor extends Processor { /** * Should permissions be changed for the newly created dirs. */ - boolean chmod = false; + protected boolean chmod = false; public boolean getChmod() { return chmod; } @@ -128,7 +128,7 @@ public class Kw3WriterProcessor extends Processor { * What should the permissions be set to. Given as three octal digits, as to * the UNIX 'chmod' command. Ex. 777 for all permissions to everyone. */ - String chmodValue = "777"; + protected String chmodValue = "777"; public String getChmodValue() { return this.chmodValue; } @@ -146,7 +146,7 @@ public class Kw3WriterProcessor extends Processor { /** * Name of collection. */ - String collection = "kw3"; + protected String collection = "kw3"; public String getCollection() { return this.collection; } @@ -157,7 +157,7 @@ public class Kw3WriterProcessor extends Processor { /** * Name of the harvester that is used for the web harvesting. */ - String harvester = "heritrix"; + protected String harvester = "heritrix"; public String getHarvester() { return this.harvester; } diff --git a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java index f487be94..0ae482e2 100644 --- a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java @@ -116,7 +116,7 @@ public class MirrorWriterProcessor extends Processor { * True if the file system is case-sensitive, like UNIX. False if the file * system is case-insensitive, like Macintosh HFS+ and Windows. */ - boolean caseSensitiveFilesystem = true; + protected boolean caseSensitiveFilesystem = true; public boolean getCaseSensitiveFilesystem() { return this.caseSensitiveFilesystem; } @@ -132,7 +132,7 @@ public class MirrorWriterProcessor extends Processor { * recommended value is [' ' %%20 " %%22 * %%2A : %%3A < %%3C \\> %%3E ? * %%3F \\\\ %%5C ^ %%5E | %%7C]. */ - List characterMap = new ArrayList(); + protected List characterMap = new ArrayList(); public List getCharacterMap() { return this.characterMap; } @@ -150,7 +150,7 @@ public class MirrorWriterProcessor extends Processor { * the URI. For example, to force all HTML files to have the same suffix, * use [text/html html]. */ - List contentTypeMap = new ArrayList(); + protected List contentTypeMap = new ArrayList(); public List getContentTypeMap() { return this.contentTypeMap; } @@ -161,7 +161,7 @@ public class MirrorWriterProcessor extends Processor { /** * If a segment starts with '.', the '.' is replaced by this. */ - String dotBegin = "%2E"; + protected String dotBegin = "%2E"; public String getDotBegin() { return this.dotBegin; } @@ -181,7 +181,7 @@ public class MirrorWriterProcessor extends Processor { * systems except Windows, '.' is recommended. For Windows, %%2E is * recommended. */ - String dotEnd = "."; + protected String dotEnd = "."; public String getDotEnd() { return this.dotEnd; } @@ -193,7 +193,7 @@ public class MirrorWriterProcessor extends Processor { /** * Implicitly append this to a URI ending with '/'. */ - String directoryFile = "index.html"; + protected String directoryFile = "index.html"; public String getDirectoryFile() { return this.directoryFile; } @@ -205,7 +205,7 @@ public class MirrorWriterProcessor extends Processor { /** * Create a subdirectory named for the host in the URI. */ - boolean createHostDirectory = true; + protected boolean createHostDirectory = true; public boolean getCreateHostDirectory() { return this.createHostDirectory; } @@ -219,7 +219,7 @@ public class MirrorWriterProcessor extends Processor { * pair. This can be used for consistency when several names are used for * one host, for example [12.34.56.78 www42.foo.com]. */ - List hostMap = new ArrayList(); + protected List hostMap = new ArrayList(); public List getHostMap() { return this.hostMap; } @@ -231,7 +231,7 @@ public class MirrorWriterProcessor extends Processor { /** * Maximum file system path length. */ - int maxPathLength = 1023; + protected int maxPathLength = 1023; public int getMaxPathLength() { return maxPathLength; } @@ -242,7 +242,7 @@ public class MirrorWriterProcessor extends Processor { /** * Maximum file system path segment length. */ - int maxSegLength = 255; + protected int maxSegLength = 255; public int getMaxSegLength() { return maxSegLength; } @@ -257,7 +257,7 @@ public class MirrorWriterProcessor extends Processor { /** * Top-level directory for mirror files. */ - String path = "mirror"; + protected String path = "mirror"; public String getPath() { return this.path; } @@ -268,7 +268,7 @@ public class MirrorWriterProcessor extends Processor { /** * Create a subdirectory named for the port in the URI. */ - boolean createPortDirectory = false; + protected boolean createPortDirectory = false; public boolean getCreatePortDirectory() { return this.createPortDirectory; } @@ -280,7 +280,7 @@ public class MirrorWriterProcessor extends Processor { * If true, the suffix is placed at the end of the path, after the query (if * any). If false, the suffix is placed before the query. */ - boolean suffixAtEnd = true; + protected boolean suffixAtEnd = true; public boolean getSuffixAtEnd() { return this.suffixAtEnd; } @@ -293,7 +293,7 @@ public class MirrorWriterProcessor extends Processor { * exceeding, the file system maximum path length, then they are all * replaced by this. */ - String tooLongDirectory = "LONG"; + protected String tooLongDirectory = "LONG"; public String getTooLongDirectory() { return this.tooLongDirectory; } @@ -309,7 +309,7 @@ public class MirrorWriterProcessor extends Processor { * com4 com5 com6 com7 com8 com9 lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 * lpt9 con nul prn]. */ - List underscoreSet = new ArrayList(); + protected List underscoreSet = new ArrayList(); public List getUnderscoreSet() { return this.underscoreSet; } diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 3b661d62..40dc78ab 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -173,7 +173,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit /** * Generator for record IDs */ - RecordIDGenerator generator = new UUIDGenerator(); + protected RecordIDGenerator generator = new UUIDGenerator(); public RecordIDGenerator getRecordIDGenerator() { return generator; } diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 87fbccf2..09dd6d1d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -70,7 +70,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * Whether to gzip-compress files when writing to disk; * by default true, meaning do-compress. */ - boolean compress = true; + protected boolean compress = true; public boolean getCompress() { return compress; } @@ -83,7 +83,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * template (below) as the 'prefix' variable for possible interpolation. * In the default/recommended naming formula, the prefix will appear first. */ - String prefix = WriterPoolMember.DEFAULT_PREFIX; + protected String prefix = WriterPoolMember.DEFAULT_PREFIX; public String getPrefix() { return prefix; } @@ -109,7 +109,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * that ARCs/WARCs are carefully named to preserve uniqueness. * */ - String template = WriterPoolMember.DEFAULT_TEMPLATE; + protected String template = WriterPoolMember.DEFAULT_TEMPLATE; public String getTemplate() { return template; } @@ -120,8 +120,8 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { /** * Max size of each file. */ - long maxFileSizeBytes = getDefaultMaxFileSize(); - abstract long getDefaultMaxFileSize(); + protected long maxFileSizeBytes = getDefaultMaxFileSize(); + protected abstract long getDefaultMaxFileSize(); public long getMaxFileSizeBytes() { return maxFileSizeBytes; } @@ -133,7 +133,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * Maximum active files in pool. This setting cannot be varied over the life * of a crawl. */ - int poolMaxActive = WriterPool.DEFAULT_MAX_ACTIVE; + protected int poolMaxActive = WriterPool.DEFAULT_MAX_ACTIVE; public int getPoolMaxActive() { return poolMaxActive; } @@ -145,7 +145,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * Maximum time to wait on idle writer before (possibly) creating an * additional instance. */ - int maxWaitForIdleMs = WriterPool.DEFAULT_MAX_WAIT_FOR_IDLE; + protected int maxWaitForIdleMs = WriterPool.DEFAULT_MAX_WAIT_FOR_IDLE; public int getMaxWaitForIdleMs() { return maxWaitForIdleMs; } @@ -162,7 +162,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * skipped entirely). * Default is false. */ - boolean skipIdenticalDigests = false; + protected boolean skipIdenticalDigests = false; public boolean getSkipIdenticalDigests() { return skipIdenticalDigests; } @@ -180,7 +180,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * exceeded this limit, this processor will stop the crawler. A value of * zero means no upper limit. */ - long maxTotalBytesToWrite = 0L; + protected long maxTotalBytesToWrite = 0L; public long getMaxTotalBytesToWrite() { return maxTotalBytesToWrite; } @@ -244,8 +244,8 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { * setting is safe to change midcrawl (You can remove and add new * dirs as the crawler progresses). */ - List storePaths = getDefaultStorePaths(); - abstract List getDefaultStorePaths(); + protected List storePaths = getDefaultStorePaths(); + protected abstract List getDefaultStorePaths(); public List getStorePaths() { return storePaths; }